fix: improve MCP tool descriptions and evaluations (#2099)

Co-authored-by: Hugo Richard <hugo.richard@vercel.com>
Co-authored-by: Sébastien Chopin <atinux@gmail.com>

Author: 3 people

.env.example

@@ -26,3 +26,7 @@ NUXT_OAUTH_GITHUB_CLIENT_SECRET=
 # Session encryption password (generate a secure random string)
 # You can use: openssl rand -base64 32
 NUXT_SESSION_PASSWORD=
+
+# MCP Evaluation (Optional - for running evalite tests)
+OPENAI_API_KEY=
+MCP_URL=http://localhost:3000/mcp

README.md

@@ -65,6 +65,34 @@ Build the application for production:
 pnpm generate
 ```
 
+### Running Evals for the MCP Server
+
+To run the evals for the MCP server, follow these steps:
+
+1. **Ensure your development server is running**  
+   Start the local Nuxt development server:
+   ```bash
+   pnpm dev
+   ```
+
+2. **Create an AI Gateway API key**  
+   Go to https://vercel.com/ai-gateway and create an API key.
+   Add the following variable to your `.env` file (replace `sk-...` with your actual key):
+   ```
+   AI_GATEWAY_API_KEY=<you-api-key>
+   ```
+
+3. **Run the evals**  
+   You can execute the evals from the command line:
+   ```bash
+   pnpm eval
+   ```
+
+   Or launch the interactive UI to run them via a web interface:
+   ```bash
+   pnpm eval:ui
+   ```
+
 ## License
 
 [MIT License](./LICENSE)

evalite.config.ts

@@ -0,0 +1,6 @@
+import 'dotenv/config'
+import { defineConfig } from 'evalite/config'
+
+export default defineConfig({
+  cache: true
+})

package.json

@@ -10,6 +10,8 @@
     "lint": "eslint . --cache",
     "typecheck": "nuxt typecheck",
     "test": "pnpm lint && pnpm typecheck",
+    "eval": "evalite",
+    "eval:ui": "evalite serve",
     "db:generate": "nuxt hub database generate",
     "db:migrate": "nuxt hub database migrate"
   },
@@ -54,15 +56,19 @@
     "valibot": "1.1.0"
   },
   "devDependencies": {
+    "@ai-sdk/mcp": "^0.0.8",
+    "@ai-sdk/openai": "3.0.0-beta.60",
     "@iconify-json/vscode-icons": "^1.2.36",
     "@nuxt/eslint": "^1.10.0",
     "@nuxt/test-utils": "^3.20.1",
     "@nuxtjs/turnstile": "^1.1.1",
     "@testing-library/vue": "^8.1.0",
     "@types/youtube": "^0.1.2",
+    "ai": "6.0.0-beta.99",
     "capture-website": "^5.1.0",
     "drizzle-kit": "^0.31.7",
     "eslint": "^9.39.1",
+    "evalite": "1.0.0-beta.13",
     "nuxt-content-twoslash": "0.1.2",
     "shiki": "^3.15.0",
     "twoslash": "^0.3.4",