Add mcpclient for evaluation sub package to call the mobile-web mcp tool (#20)

* feat: setup basic evaluation frame work

* feat: setup mobileWebMcpClient for evaluation

* move README.md up to root of evaluation sub package

* fix a test failure

Author: ben-zhang-at-salesforce

.codecov.yml

@@ -67,5 +67,3 @@ parsers:
       loop: yes
       method: no
       macro: no
-
-      macro: no

.github/workflows/run-tests.yml

@@ -16,6 +16,7 @@ jobs:
           node-version: ${{ matrix.node }}
           cache: 'npm'
       - run: npm install
+      - run: npm run build:all  
       - run: npm run test:coverage
       # Only run Xvfb cleanup on Ubuntu (Unix systems)
       - if: matrix.os == 'ubuntu-latest'
@@ -30,6 +31,13 @@ jobs:
           flags: mobile-web
           name: mobile-web-coverage
 
+      - name: Upload evaluation coverage
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: packages/evaluation/coverage/*.json
+          flags: evaluation
+          name: evaluation-coverage
       # Add more packages as you create them
       # - name: Upload backend-api coverage
       #   uses: codecov/codecov-action@v3

package-lock.json

@@ -29,9 +29,14 @@
     "url": "https://github.com/forcedotcom/mobile-mcp-tools/issues"
   },
   "homepage": "https://github.com/forcedotcom/mobile-mcp-tools#readme",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.12.3"
+  },
   "devDependencies": {
     "@eslint/js": "^9.30.0",
     "@nx/workspace": "^21.2.1",
+    "@types/node": "^24.0.10",
+    "@vitest/coverage-v8": "^3.2.3",
     "@typescript-eslint/eslint-plugin": "^8.35.1",
     "eslint": "^9.30.0",
     "eslint-config-prettier": "^10.1.5",
@@ -45,7 +50,7 @@
     "typescript": "^5.8.3",
     "typescript-eslint": "^8.35.1",
     "vite": "^6.3.5",
-    "zod": "^3.25.0"
+    "zod": "^3.25.67"
   },
   "nx": {
     "extends": "@nx/workspace/presets/npm.json"

package.json

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2025, salesforce.com, inc.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ **/
+
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+
+// Path to the MCP server entry point
+const SERVER_COMMAND = 'ts-node';
+const SERVER_ARGS = [require.resolve('../../../mobile-web/dist/index.js')];
+
+export class MobileWebMcpClient {
+  private client: Client;
+  private transport: StdioClientTransport;
+
+  constructor() {
+    this.transport = new StdioClientTransport({
+      command: SERVER_COMMAND,
+      args: SERVER_ARGS,
+    });
+    this.client = new Client({
+      name: 'MobileWebMcpClient',
+      version: '1.0.0',
+    });
+  }
+
+  async connect() {
+    await this.client.connect(this.transport);
+  }
+
+  async disconnect() {
+    await this.client.close();
+  }
+
+  async listTools() {
+    return this.client.listTools();
+  }
+
+  async callTool(toolName: string, params: Record<string, any>) {
+    return this.client.callTool({ name: toolName, arguments: params });
+  }
+}

packages/evaluation/src/mcpclient/mobileWebMcpClient.ts

@@ -0,0 +1,66 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { MobileWebMcpClient } from '../../src/mcpclient/mobileWebMcpClient.js';
+import { spawn } from 'child_process';
+import { execSync } from 'child_process';
+import path from 'path';
+
+// Integration tests against the real MCP server
+describe('MobileWebMcpClient (integration)', () => {
+  let client: MobileWebMcpClient;
+  let serverProcess: import('child_process').ChildProcess;
+
+  beforeAll(async () => {
+    // Start the MCP server
+    serverProcess = spawn('npm', ['run', 'mobile-web:server:start'], {
+      cwd: path.resolve(__dirname, '../../../..'),
+      stdio: 'inherit',
+      shell: true,
+    });
+
+    // Wait for the server to be ready (simple delay, adjust as needed)
+    await new Promise(resolve => setTimeout(resolve, 5000));
+
+    client = new MobileWebMcpClient();
+    await client.connect();
+  });
+
+  afterAll(async () => {
+    if (client) {
+      await client.disconnect();
+    }
+    if (serverProcess) {
+      serverProcess.kill();
+    }
+  });
+
+  const toolNameToGroundingTitleMapping: Record<string, string> = {
+    'sfmobile-web-app-review': 'App Review Service Grounding Context',
+    'sfmobile-web-ar-space-capture': 'AR Space Capture Service Grounding Context',
+    'sfmobile-web-barcode-scanner': 'Barcode Scanner Service Grounding Context',
+    'sfmobile-web-biometrics': 'Biometrics Service Grounding Context',
+    'sfmobile-web-calendar': 'Calendar Service Grounding Context',
+    'sfmobile-web-contacts': 'Contacts Service Grounding Context',
+    'sfmobile-web-document-scanner': 'Document Scanner Service Grounding Context',
+    'sfmobile-web-geofencing': 'Geofencing Service Grounding Context',
+    'sfmobile-web-location': 'Location Service Grounding Context',
+    'sfmobile-web-nfc': 'NFC Service Grounding Context',
+    'sfmobile-web-payments': 'Payments Service Grounding Context',
+  };
+
+  it('should list available tools', async () => {
+    const result = await client.listTools();
+    const toolNames = result.tools.map(tool => tool.name);
+    expect(toolNames).toEqual(expect.arrayContaining(Object.keys(toolNameToGroundingTitleMapping)));
+  });
+
+  // Test each tool with the correct grounding context
+  it.each(Object.keys(toolNameToGroundingTitleMapping))(
+    'should call a tool %s and return correct grounding context',
+    async toolName => {
+      const groundTitle = toolNameToGroundingTitleMapping[toolName];
+      const result = await client.callTool(toolName, {});
+      const groundingContext = result.content?.[0]?.text;
+      expect(groundingContext).toEqual(expect.stringContaining(groundTitle));
+    }
+  );
+});