diff --git a/README.md b/README.md index 8977bbf..4ec3f80 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ A Model Context Protocol (MCP) server for Genesys Cloud's Platform API. ## Usage with Claude Desktop -Add this to your `claude_desktop_config.json`: - ### NPX +Add this to your `claude_desktop_config.json`: + ```json { "mcpServers": { @@ -41,6 +41,39 @@ Add this to your `claude_desktop_config.json`: } ``` +### Desktop Extension + +This MCP Server provides a Desktop Extension (.dxt file) along with each [release](https://github.com/MakingChatbots/genesys-cloud-mcp-server/releases), +which is a single-click installable package for Claude Desktop. To use it: + +1. Download the `.dxt` file from the [latest release](https://github.com/MakingChatbots/genesys-cloud-mcp-server/releases) +2. In Claude Desktop navigate to Settings > Extensions. +3. Browse to, or drag in the .dxt file downloaded +4. Click "Install" +5. Configure the Region and OAuth Client for the extension + +The extension will now be available in your conversations. + +## Usage with Gemini CLI + +Add below to your `.gemini/settings.json` file. You can read more about the [setup from the official guide](https://github.com/google-gemini/gemini-cli/blob/main/docs/cli/tutorials.md#configure-the-mcp-server-in-settingsjson). + +```json +{ + "mcpServers": { + "genesysCloud": { + "command": "npx", + "args": ["-y", "@makingchatbots/genesys-cloud-mcp-server"], + "env": { + "GENESYSCLOUD_REGION": "${GENESYSCLOUD_REGION}", + "GENESYSCLOUD_OAUTHCLIENT_ID": "${GENESYSCLOUD_OAUTHCLIENT_ID}", + "GENESYSCLOUD_OAUTHCLIENT_SECRET": "${GENESYSCLOUD_OAUTHCLIENT_SECRET}" + } + } + } +} +``` + ## Authentication This currently only supports a stdio server. To configure authentication you'll need to: diff --git a/package-lock.json b/package-lock.json index a6f594a..db2bfcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@modelcontextprotocol/sdk": "^1.16.0", "date-fns": "^4.1.0", + "dotenv": "^17.2.1", "purecloud-platform-client-v2": "^227.0.0", "zod": "^3.23.8" }, @@ -21,6 +22,7 @@ "@anthropic-ai/dxt": "^0.2.5", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.31.0", + "@google/genai": "^1.12.0", "@types/eslint-config-prettier": "^6.11.3", "@types/node": "^22.16.4", "@typescript-eslint/eslint-plugin": "^8.33.1", @@ -684,6 +686,28 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@google/genai": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.12.0.tgz", + "integrity": "sha512-JBkQsULVexdM9zY4iXbm3A2dJ7El/hSPGCnxuRWPJNgeqcfYuyUnPTSy+I/v+MvTbz/occVmONSD6wn+17QLkg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^9.14.2", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.11.0" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -2256,6 +2280,16 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2523,6 +2557,37 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/body-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", @@ -2567,6 +2632,13 @@ "node": ">=8" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3096,6 +3168,18 @@ "node": ">=0.10.0" } }, + "node_modules/dotenv": { + "version": "17.2.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz", + "integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -3110,6 +3194,16 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -3913,6 +4007,13 @@ "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true, + "license": "MIT" + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -4291,6 +4392,38 @@ "node": ">= 12" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/get-east-asian-width": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz", @@ -4419,6 +4552,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -4445,6 +4606,20 @@ "dev": true, "license": "MIT" }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dev": true, + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/has-bigints": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.1.0.tgz", @@ -4557,6 +4732,20 @@ "node": ">= 0.8" } }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -5138,6 +5327,16 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -5186,6 +5385,29 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -5609,6 +5831,27 @@ "node": ">= 0.6" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-forge": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", @@ -7099,6 +7342,13 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/triple-beam": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", @@ -7423,6 +7673,20 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -7644,6 +7908,24 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -7853,6 +8135,28 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/yaml": { "version": "2.8.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", diff --git a/package.json b/package.json index 0b85d8a..78de7b2 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "lint:fix": "eslint --cache --fix --ext .ts ./src", "check": "npm run lint:check && npm run prettier:check", "fix": "npm run lint:fix && npm run prettier:fix", - "test": "npx vitest --config ./vitest.config.ts", + "test": "npx vitest --config ./vitest.config.ts --project unit", + "test:evaluation": "npx vitest --config ./vitest.config.ts --project evaluation", "prepublishOnly": "npm run test && npm run build", "test:pack": "npm run build && npm pack --pack-destination ./dist" }, @@ -47,8 +48,11 @@ "zod": "^3.23.8" }, "devDependencies": { + "dotenv": "^17.2.1", + "@anthropic-ai/dxt": "^0.2.5", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.31.0", + "@google/genai": "^1.12.0", "@types/eslint-config-prettier": "^6.11.3", "@types/node": "^22.16.4", "@typescript-eslint/eslint-plugin": "^8.33.1", @@ -58,7 +62,6 @@ "eslint-import-resolver-typescript": "^4.4.4", "lint-staged": "^16.1.2", "prettier": "^3.6.2", - "@anthropic-ai/dxt": "^0.2.5", "tsx": "^4.20.3", "typescript": "^5.8.3", "typescript-eslint": "^8.37.0", diff --git a/src/tools/searchQueues.ts b/src/tools/searchQueues.ts index 4857ef0..bfd3a3a 100644 --- a/src/tools/searchQueues.ts +++ b/src/tools/searchQueues.ts @@ -17,6 +17,21 @@ export interface ToolDependencies { readonly routingApi: Pick; } +export interface SearchQueuesResponse { + queues: { + name: string; + id: string; + description?: string; + memberCount?: number; + }[]; + pagination: { + totalMatchingQueues: string | number; + pageNumber: string | number; + pageSize: string | number; + totalPages: string | number; + }; +} + function formatQueuesJson( queues: PartRequired[], pagination: { @@ -25,7 +40,7 @@ function formatQueuesJson( pageCount?: number; totalHits?: number; }, -): Record { +): SearchQueuesResponse { return { queues: queues.map((q) => ({ name: q.name, @@ -94,18 +109,18 @@ export const searchQueues: ToolFactory< const foundQueues = entities.filter(hasIdAndName); + const response: SearchQueuesResponse = formatQueuesJson(foundQueues, { + pageNumber: entities.length === 0 ? 0 : result.pageNumber, + pageSize: entities.length === 0 ? 0 : result.pageSize, + pageCount: entities.length === 0 ? 0 : result.pageCount, + totalHits: entities.length === 0 ? 0 : result.total, + }); + return { content: [ { type: "text", - text: JSON.stringify( - formatQueuesJson(foundQueues, { - pageNumber: entities.length === 0 ? 0 : result.pageNumber, - pageSize: entities.length === 0 ? 0 : result.pageSize, - pageCount: entities.length === 0 ? 0 : result.pageCount, - totalHits: entities.length === 0 ? 0 : result.total, - }), - ), + text: JSON.stringify(response), }, ], }; diff --git a/src/tools/utils/paginationSection.ts b/src/tools/utils/paginationSection.ts index a0f5b84..8036029 100644 --- a/src/tools/utils/paginationSection.ts +++ b/src/tools/utils/paginationSection.ts @@ -14,10 +14,16 @@ const calculateTotalPages = ( return Math.ceil(totalHits / pageSize); }; -export function paginationSection( - totalSectionName: string, +type PaginationResult = Record & { + pageNumber: string | number; + pageSize: string | number; + totalPages: string | number; +}; + +export function paginationSection( + totalSectionName: T, { pageSize, pageNumber, totalHits, pageCount }: PaginationArgs, -) { +): PaginationResult { let formattedTotalPages: string | number = "N/A"; if (pageCount !== undefined) { formattedTotalPages = pageCount; @@ -30,5 +36,5 @@ export function paginationSection( pageSize: pageSize ?? "N/A", totalPages: formattedTotalPages, [totalSectionName]: totalHits ?? "N/A", - }; + } as PaginationResult; } diff --git a/tests/evaluations/gemini-tool-efficiency.test.ts b/tests/evaluations/gemini-tool-efficiency.test.ts new file mode 100644 index 0000000..5829a91 --- /dev/null +++ b/tests/evaluations/gemini-tool-efficiency.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { GoogleGenAI, mcpToTool } from "@google/genai"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { type CallToolRequest } from "@modelcontextprotocol/sdk/types.js"; +import { type SearchQueuesResponse } from "../../src/tools/searchQueues.js"; +import { + type CallToolInterception, + type CallToolResult, + callToolInterceptor, + spyOnClientCallTool, + prettyPrintResults, + toolUsageEvaluator, +} from "./utils"; + +const timeout = 30_000; +const model = "gemini-2.5-flash"; + +const searchQueueInterceptor: CallToolInterception = { + shouldIntercept: (call: CallToolRequest["params"]) => { + return call.name === "search_queues"; + }, + intercept: (): Promise => { + const response: SearchQueuesResponse = { + queues: [ + { + name: "Lucas_Test_Queue", + id: "c1c893ed-b765-4361-abc3-7bb3f5b7d6e5", + memberCount: 0, + }, + ], + pagination: { + pageNumber: 1, + pageSize: 100, + totalPages: 1, + totalMatchingQueues: 1, + }, + }; + + return Promise.resolve({ + content: [ + { + type: "text", + text: JSON.stringify(response), + }, + ], + }); + }, +}; + +const interceptors = [searchQueueInterceptor]; + +describe( + "Evaluate efficient tool use", + () => { + let client: Client; + let ai: GoogleGenAI; + + beforeEach(async () => { + const transport = new StdioClientTransport({ + command: "npm", + args: ["run", "start"], + env: { + GENESYSCLOUD_REGION: process.env.GENESYSCLOUD_REGION!, + GENESYSCLOUD_OAUTHCLIENT_ID: process.env.GENESYSCLOUD_OAUTHCLIENT_ID!, + GENESYSCLOUD_OAUTHCLIENT_SECRET: + process.env.GENESYSCLOUD_OAUTHCLIENT_SECRET!, + }, + }); + + client = new Client({ + name: "genesys-cloud-mcp-client", + version: "1.0.0", + }); + + ai = new GoogleGenAI({}); + await client.connect(transport); + }); + + afterEach(async () => client.close()); + + test("search_queue is called for simple queue query", async () => { + const interceptor = callToolInterceptor(interceptors); + + spyOnClientCallTool(client).spy.mockImplementation((args) => + interceptor.call(args), + ); + + const response = await ai.models.generateContent({ + model, + contents: `How many queues start with LUCAS?`, + config: { + tools: [mcpToTool(client)], + }, + }); + + console.log("LLM Response", response.text); + + const evaluationResults = toolUsageEvaluator(interceptor.toolCallTraces, [ + "search_queues", + ]); + + console.log("Evaluation Results", prettyPrintResults(evaluationResults)); + + expect(evaluationResults.toolAccuracy).toBe(1); + expect(evaluationResults.extraneousToolsCalled.length).toBe(0); + }); + + test("Total conversations last week", async () => { + const clientCallToolSpyResult = spyOnClientCallTool(client); + + const response = await ai.models.generateContent({ + model, + contents: `How many conversations were there between today (${new Date().toISOString()}) and 15 days ago against queues containing TEST in the name?`, + config: { + tools: [mcpToTool(client)], + }, + }); + + console.log("LLM Response", response.text); + console.dir(await clientCallToolSpyResult.tracesFromCallTools(), { + depth: 10, + }); + }); + }, + timeout, +); diff --git a/tests/evaluations/utils/callToolInterceptor.ts b/tests/evaluations/utils/callToolInterceptor.ts new file mode 100644 index 0000000..5c9bd7c --- /dev/null +++ b/tests/evaluations/utils/callToolInterceptor.ts @@ -0,0 +1,49 @@ +import { z } from "zod"; +import { + type CallToolRequest, + CallToolResultSchema, +} from "@modelcontextprotocol/sdk/types.js"; + +export type CallToolRequestParams = CallToolRequest["params"]; +export type CallToolResult = z.infer; + +export interface CallToolTrace { + request: CallToolRequestParams; + result: CallToolResult; +} + +export interface CallToolInterception { + shouldIntercept: (call: CallToolRequestParams) => T; + intercept: (call: CallToolRequestParams) => Promise; +} + +const errorThrowingInterceptor: CallToolInterception = { + shouldIntercept: () => true, + intercept: () => { + throw new Error("No interceptor found"); + }, +}; + +export function callToolInterceptor( + interceptors: CallToolInterception[], + defaultInterceptor: CallToolInterception = errorThrowingInterceptor, +) { + const toolCallTraces: CallToolTrace[] = []; + + return { + async call(request: CallToolRequestParams): Promise { + const interceptor = + interceptors.find((i) => i.shouldIntercept(request)) ?? + defaultInterceptor; + + const result = await interceptor.intercept(request); + toolCallTraces.push({ request, result }); + + return result; + }, + + get toolCallTraces(): readonly CallToolTrace[] { + return toolCallTraces; + }, + }; +} diff --git a/tests/evaluations/utils/index.ts b/tests/evaluations/utils/index.ts new file mode 100644 index 0000000..6882f5e --- /dev/null +++ b/tests/evaluations/utils/index.ts @@ -0,0 +1,10 @@ +export { spyOnClientCallTool } from "./spyOnClientCallTool.js"; +export { + type CallToolInterception, + callToolInterceptor, + type CallToolResult, +} from "./callToolInterceptor.js"; +export { + prettyPrintResults, + toolUsageEvaluator, +} from "./toolUsageEvaluator.js"; diff --git a/tests/evaluations/utils/spyOnClientCallTool.ts b/tests/evaluations/utils/spyOnClientCallTool.ts new file mode 100644 index 0000000..d53e176 --- /dev/null +++ b/tests/evaluations/utils/spyOnClientCallTool.ts @@ -0,0 +1,26 @@ +import { vi } from "vitest"; +import { + type CallToolRequestParams, + type CallToolTrace, +} from "./callToolInterceptor.js"; +import { CallToolResultSchema } from "@modelcontextprotocol/sdk/types.js"; +import { type Client } from "@modelcontextprotocol/sdk/client/index.js"; + +export function spyOnClientCallTool(client: Client) { + const spy = vi.spyOn(client, "callTool"); + + return { + spy, + tracesFromCallTools: (): Promise => + Promise.all( + spy.mock.calls.map(async (args, i) => { + const request: CallToolRequestParams = args[0]; + const result: unknown = await spy.mock.results[i]?.value; + return { + request, + result: CallToolResultSchema.parse(result), + }; + }), + ), + }; +} diff --git a/tests/evaluations/utils/toolUsageEvaluator.ts b/tests/evaluations/utils/toolUsageEvaluator.ts new file mode 100644 index 0000000..70f3051 --- /dev/null +++ b/tests/evaluations/utils/toolUsageEvaluator.ts @@ -0,0 +1,57 @@ +import { type CallToolTrace } from "./callToolInterceptor.js"; + +export function toolUsageEvaluator( + traces: readonly CallToolTrace[], + expectedTools: string[], +) { + const calledTools = traces.map((t) => t.request.name); + + return { + /** + * Tools that were expected to be used and were actually used. + */ + get correctToolsCalled() { + return expectedTools.filter((t) => calledTools.includes(t)); + }, + + /** + * Tools that were expected but not used. + */ + get missingTools() { + return expectedTools.filter((t) => !calledTools.includes(t)); + }, + + /** + * Tools that were used but not expected. + */ + get extraneousToolsCalled() { + return calledTools.filter((t) => !expectedTools.includes(t)); + }, + + /** + * Total number of tool calls made by the LLM during the task. + */ + get totalCalls() { + return traces.length; + }, + + /** + * Proportion of expected tools that were correctly used (0–1) + */ + get toolAccuracy() { + return this.correctToolsCalled.length / expectedTools.length; + }, + }; +} + +export function prettyPrintResults( + evaluationResults: ReturnType, +) { + return { + "Correct tools:": evaluationResults.correctToolsCalled, + "Extraneous calls:": evaluationResults.extraneousToolsCalled, + "Missing tools:": evaluationResults.missingTools, + "Total calls:": evaluationResults.totalCalls, + "Tool accuracy:": evaluationResults.toolAccuracy, + }; +} diff --git a/tests/serverRuns.test.ts b/tests/integration/serverRuns.test.ts similarity index 91% rename from tests/serverRuns.test.ts rename to tests/integration/serverRuns.test.ts index 7697b1e..b9620d6 100644 --- a/tests/serverRuns.test.ts +++ b/tests/integration/serverRuns.test.ts @@ -4,7 +4,7 @@ import { execSync } from "node:child_process"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; import { type CallToolResult } from "@modelcontextprotocol/sdk/types.js"; -import packageInfo from "../package.json" with { type: "json" }; +import packageInfo from "../../package.json" with { type: "json" }; describe("Server Runs", () => { let client: Client | null = null; @@ -20,7 +20,7 @@ describe("Server Runs", () => { test("server returns list of tools", async () => { const transport = new StdioClientTransport({ command: "node", - args: ["--inspect", join(__dirname, "../dist/index.js")], + args: ["--inspect", join(__dirname, "../../dist/index.js")], env: { // Provides path for node binary to be used in test PATH: process.env.PATH!, @@ -50,7 +50,7 @@ describe("Server Runs", () => { test("server version matches version in package.json", async () => { const transport = new StdioClientTransport({ command: "node", - args: ["--inspect", join(__dirname, "../dist/index.js")], + args: ["--inspect", join(__dirname, "../../dist/index.js")], env: { // Provides path for node binary to be used in test PATH: process.env.PATH!, @@ -71,7 +71,7 @@ describe("Server Runs", () => { test("server runs via cli", async () => { const transport = new StdioClientTransport({ command: "node", - args: ["--inspect", join(__dirname, "../dist/cli.js")], + args: ["--inspect", join(__dirname, "../../dist/cli.js")], env: { PATH: process.env.PATH!, }, @@ -112,7 +112,7 @@ describe("Server Runs", () => { test("calling tool errors if not OAuth config", async () => { const transport = new StdioClientTransport({ command: "node", - args: ["--inspect", join(__dirname, "../dist/cli.js")], + args: ["--inspect", join(__dirname, "../../dist/cli.js")], env: { PATH: process.env.PATH!, }, diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..ef604be --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.json", + "include": ["src", "tests"], + "compilerOptions": { + "rootDir": "." + } +} diff --git a/vitest.config.ts b/vitest.config.ts index 4275737..8e8e8dd 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,8 +3,29 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", - include: ["src/**/*.test.ts", "tests/*.test.ts"], isolate: false, watch: false, + + projects: [ + { + test: { + name: "unit", + include: ["src/**/*.test.ts", "tests/integration/**/*.test.ts"], + typecheck: { + tsconfig: "tsconfig.test.json", + }, + }, + }, + { + test: { + name: "evaluation", + include: ["tests/evaluations/**/*.test.ts"], + setupFiles: ["dotenv/config"], + typecheck: { + tsconfig: "tsconfig.test.json", + }, + }, + }, + ], }, });