Skip to content

Commit 78298c3

Browse files
pavelfeldmanpavelfeldman
authored
chore: introduce verification tools (microsoft#951)
1 parent 7774ad9 commit 78298c3

File tree

5 files changed

+680
-4
lines changed

5 files changed

+680
-4
lines changed

‎config.d.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
import type * as playwright from 'playwright';
1818

19-
export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf';
19+
export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf' | 'verify';
2020

2121
export type Config = {
2222
/**

‎src/tools.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,15 @@ import files from './tools/files.js';
2222
import form from './tools/form.js';
2323
import install from './tools/install.js';
2424
import keyboard from './tools/keyboard.js';
25+
import mouse from './tools/mouse.js';
2526
import navigate from './tools/navigate.js';
2627
import network from './tools/network.js';
2728
import pdf from './tools/pdf.js';
2829
import snapshot from './tools/snapshot.js';
2930
import tabs from './tools/tabs.js';
3031
import screenshot from './tools/screenshot.js';
3132
import wait from './tools/wait.js';
32-
import mouse from './tools/mouse.js';
33+
import verify from './tools/verify.js';
3334

3435
import type { Tool } from './tools/tool.js';
3536
import type { FullConfig } from './config.js';
@@ -51,6 +52,7 @@ export const allTools: Tool<any>[] = [
5152
...snapshot,
5253
...tabs,
5354
...wait,
55+
...verify,
5456
];
5557

5658
export function filteredTools(config: FullConfig) {

‎src/tools/verify.ts

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/**
2+
* Copyright (c) Microsoft Corporation.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import { z } from 'zod';
18+
19+
import { defineTabTool } from './tool.js';
20+
import * as javascript from '../utils/codegen.js';
21+
import { generateLocator } from './utils.js';
22+
23+
const verifyElement = defineTabTool({
24+
capability: 'verify',
25+
schema: {
26+
name: 'browser_verify_element_visible',
27+
title: 'Verify element visible',
28+
description: 'Verify element is visible on the page',
29+
inputSchema: z.object({
30+
role: z.string().describe('ROLE of the element. Can be found in the snapshot like this: \`- {ROLE} "Accessible Name":\`'),
31+
accessibleName: z.string().describe('ACCESSIBLE_NAME of the element. Can be found in the snapshot like this: \`- role "{ACCESSIBLE_NAME}"\`'),
32+
}),
33+
type: 'readOnly',
34+
},
35+
36+
handle: async (tab, params, response) => {
37+
const locator = tab.page.getByRole(params.role as any, { name: params.accessibleName });
38+
if (await locator.count() === 0) {
39+
response.addError(`Element with role "${params.role}" and accessible name "${params.accessibleName}" not found`);
40+
return;
41+
}
42+
43+
response.addCode(`await expect(page.getByRole(${javascript.escapeWithQuotes(params.role)}, { name: ${javascript.escapeWithQuotes(params.accessibleName)} })).toBeVisible();`);
44+
response.addResult('Done');
45+
},
46+
});
47+
48+
const verifyText = defineTabTool({
49+
capability: 'verify',
50+
schema: {
51+
name: 'browser_verify_text_visible',
52+
title: 'Verify text visible',
53+
description: `Verify text is visible on the page. Prefer ${verifyElement.schema.name} if possible.`,
54+
inputSchema: z.object({
55+
text: z.string().describe('TEXT to verify. Can be found in the snapshot like this: \`- role "Accessible Name": {TEXT}\` or like this: \`- text: {TEXT}\`'),
56+
}),
57+
type: 'readOnly',
58+
},
59+
60+
handle: async (tab, params, response) => {
61+
const locator = tab.page.getByText(params.text).filter({ visible: true });
62+
if (await locator.count() === 0) {
63+
response.addError('Text not found');
64+
return;
65+
}
66+
67+
response.addCode(`await expect(page.getByText(${javascript.escapeWithQuotes(params.text)})).toBeVisible();`);
68+
response.addResult('Done');
69+
},
70+
});
71+
72+
const verifyList = defineTabTool({
73+
capability: 'verify',
74+
schema: {
75+
name: 'browser_verify_list_visible',
76+
title: 'Verify list visible',
77+
description: 'Verify list is visible on the page',
78+
inputSchema: z.object({
79+
element: z.string().describe('Human-readable list description'),
80+
ref: z.string().describe('Exact target element reference that points to the list'),
81+
items: z.array(z.string()).describe('Items to verify'),
82+
}),
83+
type: 'readOnly',
84+
},
85+
86+
handle: async (tab, params, response) => {
87+
const locator = await tab.refLocator({ ref: params.ref, element: params.element });
88+
const itemTexts: string[] = [];
89+
for (const item of params.items) {
90+
const itemLocator = locator.getByText(item);
91+
if (await itemLocator.count() === 0) {
92+
response.addError(`Item "${item}" not found`);
93+
return;
94+
}
95+
itemTexts.push((await itemLocator.textContent())!);
96+
}
97+
const ariaSnapshot = `\`
98+
- list:
99+
${itemTexts.map(t => ` - text: ${javascript.escapeWithQuotes(t, '"')}`).join('\n')}
100+
\``;
101+
response.addCode(`await expect(page.locator('body')).toMatchAriaSnapshot(${ariaSnapshot});`);
102+
response.addResult('Done');
103+
},
104+
});
105+
106+
const verifyValue = defineTabTool({
107+
capability: 'verify',
108+
schema: {
109+
name: 'browser_verify_value',
110+
title: 'Verify value',
111+
description: 'Verify element value',
112+
inputSchema: z.object({
113+
type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the element'),
114+
element: z.string().describe('Human-readable element description'),
115+
ref: z.string().describe('Exact target element reference that points to the element'),
116+
value: z.string().describe('Value to verify. For checkbox, use "true" or "false".'),
117+
}),
118+
type: 'readOnly',
119+
},
120+
121+
handle: async (tab, params, response) => {
122+
const locator = await tab.refLocator({ ref: params.ref, element: params.element });
123+
const locatorSource = `page.${await generateLocator(locator)}`;
124+
if (params.type === 'textbox' || params.type === 'slider' || params.type === 'combobox') {
125+
const value = await locator.inputValue();
126+
if (value !== params.value) {
127+
response.addError(`Expected value "${params.value}", but got "${value}"`);
128+
return;
129+
}
130+
response.addCode(`await expect(${locatorSource}).toHaveValue(${javascript.quote(params.value)});`);
131+
} else if (params.type === 'checkbox' || params.type === 'radio') {
132+
const value = await locator.isChecked();
133+
if (value !== (params.value === 'true')) {
134+
response.addError(`Expected value "${params.value}", but got "${value}"`);
135+
return;
136+
}
137+
const matcher = value ? 'toBeChecked' : 'not.toBeChecked';
138+
response.addCode(`await expect(${locatorSource}).${matcher}();`);
139+
}
140+
response.addResult('Done');
141+
},
142+
});
143+
144+
export default [
145+
verifyElement,
146+
verifyText,
147+
verifyList,
148+
verifyValue,
149+
];

‎tests/fixtures.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
3131
import type { Stream } from 'stream';
3232

3333
export type TestOptions = {
34+
mcpArgs: string[] | undefined;
3435
mcpBrowser: string | undefined;
3536
mcpMode: 'docker' | undefined;
3637
};
@@ -65,17 +66,19 @@ type WorkerFixtures = {
6566

6667
export const test = baseTest.extend<TestFixtures & TestOptions, WorkerFixtures>({
6768

69+
mcpArgs: [undefined, { option: true }],
70+
6871
client: async ({ startClient }, use) => {
6972
const { client } = await startClient();
7073
await use(client);
7174
},
7275

73-
startClient: async ({ mcpHeadless, mcpBrowser, mcpMode }, use, testInfo) => {
76+
startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => {
7477
const configDir = path.dirname(test.info().config.configFile!);
7578
const clients: Client[] = [];
7679

7780
await use(async options => {
78-
const args: string[] = [];
81+
const args: string[] = mcpArgs ?? [];
7982
if (process.env.CI && process.platform === 'linux')
8083
args.push('--no-sandbox');
8184
if (mcpHeadless)

0 commit comments

Comments
 (0)