feat: update model benchmark schema and data

ericyangpan · ericyangpan · commit b8dfad47d838 · 2025-12-15T09:39:57.000+08:00
- Add required fields for all benchmark metrics in model schema
- Update all model manifests with complete benchmark data
  - Add mmmu, mmmuPro, webDevArena fields
  - Update existing benchmark values (terminalBench, etc.)
- Update cursor IDE manifest
diff --git a/manifests/$schemas/model.schema.json b/manifests/$schemas/model.schema.json
@@ -99,6 +99,15 @@
               "description": "LiveCodeBench Pass@1 score (percentage)"
             }
           },
+          "required": [
+            "sweBench",
+            "terminalBench",
+            "mmmu",
+            "mmmuPro",
+            "webDevArena",
+            "sciCode",
+            "liveCodeBench"
+          ],
           "additionalProperties": false
         },
         "platformUrls": {
diff --git a/manifests/ides/cursor.json b/manifests/ides/cursor.json
@@ -81,7 +81,7 @@
   "relatedProducts": [
     {
       "type": "cli",
-      "productId": "cursor-agent"
+      "productId": "cursor-cli"
     }
   ],
   "platforms": [
diff --git a/manifests/models/claude-haiku-4-5.json b/manifests/models/claude-haiku-4-5.json
@@ -25,9 +25,12 @@
   "capabilities": ["function-calling", "tool-choice", "structured-outputs", "reasoning"],
   "benchmarks": {
     "sweBench": null,
-    "terminalBench": 0.298,
+    "terminalBench": 0.355,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/claude-opus-4-1.json b/manifests/models/claude-opus-4-1.json
@@ -24,10 +24,13 @@
   "inputModalities": ["text"],
   "capabilities": ["function-calling", "tool-choice", "structured-outputs"],
   "benchmarks": {
-    "sweBench": null,
-    "terminalBench": 0.38,
+    "sweBench": 74.4,
+    "terminalBench": 0.631,
     "sciCode": null,
-    "liveCodeBench": 46.9
+    "liveCodeBench": 46.9,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/claude-opus-4.json b/manifests/models/claude-opus-4.json
@@ -24,10 +24,13 @@
   "inputModalities": ["text"],
   "capabilities": ["function-calling", "tool-choice", "structured-outputs"],
   "benchmarks": {
-    "sweBench": null,
+    "sweBench": 67.6,
     "terminalBench": 0.578,
     "sciCode": null,
-    "liveCodeBench": 56.6
+    "liveCodeBench": 56.6,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/claude-sonnet-4-5.json b/manifests/models/claude-sonnet-4-5.json
@@ -24,10 +24,13 @@
   "inputModalities": ["text"],
   "capabilities": ["function-calling", "tool-choice", "structured-outputs"],
   "benchmarks": {
-    "sweBench": null,
+    "sweBench": 70.6,
     "terminalBench": 0.428,
     "sciCode": null,
-    "liveCodeBench": 47.1
+    "liveCodeBench": 47.1,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/claude-sonnet-4.json b/manifests/models/claude-sonnet-4.json
@@ -24,10 +24,13 @@
   "inputModalities": ["text"],
   "capabilities": ["function-calling", "tool-choice", "structured-outputs", "reasoning"],
   "benchmarks": {
-    "sweBench": null,
+    "sweBench": 64.93,
     "terminalBench": 0.428,
     "sciCode": null,
-    "liveCodeBench": 55.9
+    "liveCodeBench": 55.9,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/composer.json b/manifests/models/composer.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/deepseek-r1.json b/manifests/models/deepseek-r1.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": 4.6,
-    "liveCodeBench": 73.1
+    "liveCodeBench": 73.1,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/deepseek-ai/DeepSeek-R1",
diff --git a/manifests/models/deepseek-v3-terminus.json b/manifests/models/deepseek-v3-terminus.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": 3.1,
-    "liveCodeBench": 27.2
+    "liveCodeBench": 27.2,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus",
diff --git a/manifests/models/gemini-2-5-flash.json b/manifests/models/gemini-2-5-flash.json
@@ -27,7 +27,10 @@
     "sweBench": 28.73,
     "terminalBench": 0.171,
     "sciCode": null,
-    "liveCodeBench": 61.9
+    "liveCodeBench": 61.9,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gemini-2-5-pro.json b/manifests/models/gemini-2-5-pro.json
@@ -27,7 +27,10 @@
     "sweBench": 53.6,
     "terminalBench": 0.326,
     "sciCode": null,
-    "liveCodeBench": 73.6
+    "liveCodeBench": 73.6,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gemini-3-pro.json b/manifests/models/gemini-3-pro.json
@@ -27,7 +27,10 @@
     "sweBench": 74.2,
     "terminalBench": 0.589,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/glm-4-6.json b/manifests/models/glm-4-6.json
@@ -27,7 +27,10 @@
     "sweBench": 55.4,
     "terminalBench": 0.245,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/zai-org/GLM-4.6",
diff --git a/manifests/models/glm-4-6v.json b/manifests/models/glm-4-6v.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.245,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/zai-org/GLM-4.6V",
diff --git a/manifests/models/gpt-4-1.json b/manifests/models/gpt-4-1.json
@@ -27,7 +27,10 @@
     "sweBench": 39.58,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gpt-4o.json b/manifests/models/gpt-4o.json
@@ -25,9 +25,12 @@
   "capabilities": ["function-calling", "tool-choice", "structured-outputs", "reasoning"],
   "benchmarks": {
     "sweBench": 21.62,
-    "terminalBench": null,
+    "terminalBench": 0.401,
     "sciCode": 1.5,
-    "liveCodeBench": 29.5
+    "liveCodeBench": 29.5,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gpt-5-1-codex.json b/manifests/models/gpt-5-1-codex.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.604,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gpt-5-1.json b/manifests/models/gpt-5-1.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.47600000000000003,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gpt-5-codex.json b/manifests/models/gpt-5-codex.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.496,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/gpt-5.json b/manifests/models/gpt-5.json
@@ -24,10 +24,13 @@
   "inputModalities": ["text"],
   "capabilities": ["function-calling", "tool-choice", "structured-outputs", "reasoning"],
   "benchmarks": {
-    "sweBench": 65,
-    "terminalBench": 0.496,
+    "sweBench": 71.8,
+    "terminalBench": 0.352,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/grok-code-fast-1.json b/manifests/models/grok-code-fast-1.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.258,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/kimi-k2-0905.json b/manifests/models/kimi-k2-0905.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905",
diff --git a/manifests/models/llama-4-maverick.json b/manifests/models/llama-4-maverick.json
@@ -27,7 +27,10 @@
     "sweBench": 21.04,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/meta-llama/Llama-4-Maverick",
diff --git a/manifests/models/minimax-m2.json b/manifests/models/minimax-m2.json
@@ -27,7 +27,10 @@
     "sweBench": 61,
     "terminalBench": 0.3,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": null,
diff --git a/manifests/models/qwen3-coder-30b-a3b.json b/manifests/models/qwen3-coder-30b-a3b.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct",
diff --git a/manifests/models/qwen3-coder-480b-a35b.json b/manifests/models/qwen3-coder-480b-a35b.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": 0.254,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct",
diff --git a/manifests/models/qwen3-coder-plus.json b/manifests/models/qwen3-coder-plus.json
@@ -27,7 +27,10 @@
     "sweBench": null,
     "terminalBench": null,
     "sciCode": null,
-    "liveCodeBench": null
+    "liveCodeBench": null,
+    "mmmu": null,
+    "mmmuPro": null,
+    "webDevArena": null
   },
   "platformUrls": {
     "huggingface": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B",

Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@`
`81`	`81`	`"relatedProducts": [`
`82`	`82`	`{`
`83`	`83`	`"type": "cli",`
`84`		`- "productId": "cursor-agent"`
	`84`	`+ "productId": "cursor-cli"`
`85`	`85`	`}`
`86`	`86`	`],`
`87`	`87`	`"platforms": [`