feat(web-integration): add more method for playwright fixture (#1024)

* feat(web-integration): wrap agent methods with test.step for enhanced UI visibility

* Revert "feat(web-integration): wrap agent methods with test.step for enhanced UI visibility"

This reverts commit 13de6f4.

* feat(web-integration): add more method for playwright fixture

Author: quanru

apps/site/docs/en/integrate-with-playwright.mdx

@@ -183,6 +183,7 @@ export const test = base.extend<PlayWrightAiFixtureType>(
 - `aiInput` - Input operation
 - `aiKeyboardPress` - Keyboard operation
 - `aiScroll` - Scroll operation
+- `aiRightClick` - Right-click operation
 
 #### Query
 
@@ -197,6 +198,12 @@ export const test = base.extend<PlayWrightAiFixtureType>(
 - `aiAssert` - AI Assertion
 - `aiWaitFor` - AI Wait
 - `aiLocate` - Locate Element
+- `runYaml` - Execute YAML automation script
+- `setAIActionContext` - Set AI action context, send background knowledge to AI model when calling `agent.aiAction()`
+- `evaluateJavaScript` - Execute JavaScript in page context
+- `logScreenshot` - Log screenshot with description in report file
+- `freezePageContext` - Freeze page context
+- `unfreezePageContext` - Unfreeze page context
 
 Besides the exposed shortcut methods, if you need to call other [API](./api.mdx) provided by the agent, you can use `agentForPage` to get the `PageAgent` instance, and use `PageAgent` to call the API for interaction:
 
@@ -230,6 +237,8 @@ test('search headphone on ebay', async ({
   aiTap,
   aiScroll,
   aiWaitFor,
+  aiRightClick,
+  logScreenshot,
 }) => {
   // Use aiInput to enter search keyword
   await aiInput('Headphones', 'search box');
@@ -259,6 +268,11 @@ test('search headphone on ebay', async ({
 
   // Use aiAssert to verify filter functionality
   await aiAssert('category filter exists on the left side');
+
+  // Use logScreenshot to capture the current state
+  await logScreenshot('Search Results', {
+    content: 'Final search results for headphones',
+  });
 });
 ```
 

apps/site/docs/zh/integrate-with-playwright.mdx

@@ -183,6 +183,7 @@ export const test = base.extend<PlayWrightAiFixtureType>(
 - `aiInput` - 输入操作
 - `aiKeyboardPress` - 键盘操作
 - `aiScroll` - 滚动操作
+- `aiRightClick` - 右键点击操作
 
 #### 查询
 
@@ -197,6 +198,12 @@ export const test = base.extend<PlayWrightAiFixtureType>(
 - `aiAssert` - 断言
 - `aiWaitFor` - 等待
 - `aiLocate` - 定位元素
+- `runYaml` - 执行 YAML 自动化脚本
+- `setAIActionContext` - 设置 AI 动作上下文，在调用 `agent.aiAction()` 时，发送给 AI 模型的背景知识
+- `evaluateJavaScript` - 在页面上下文中执行 JavaScript
+- `logScreenshot` - 在报告文件中记录当前截图，并添加描述
+- `freezePageContext` - 冻结页面上下文
+- `unfreezePageContext` - 解冻页面上下文
 
 除了上述暴露的快捷方法之外，如果还需要调用其它 agent 提供的 [API](./api.mdx)，请使用 `agentForPage` 获取 `PageAgent` 实例，使用 `PageAgent` 调用 API 进行交互：
 
@@ -210,8 +217,6 @@ test('case demo', async ({ agentForPage, page }) => {
 });
 ```
 
-
-
 #### 示例代码
 
 ```typescript title="./e2e/ebay-search.spec.ts"
@@ -232,6 +237,8 @@ test('search headphone on ebay', async ({
   aiTap,
   aiScroll,
   aiWaitFor,
+  aiRightClick,
+  logScreenshot,
 }) => {
   // 使用 aiInput 输入搜索关键词
   await aiInput('Headphones', '搜索框');
@@ -262,8 +269,11 @@ test('search headphone on ebay', async ({
 
   // 使用 aiAssert 验证筛选功能
   await aiAssert('界面左侧有类目筛选功能');
+
+  // 使用 logScreenshot 记录当前状态
+  await logScreenshot('搜索结果', { content: '耳机搜索的最终结果' });
 });
-````
+```
 
 更多 Agent 的 API 讲解请参考 [API 参考](./api.mdx)。
 

packages/web-integration/src/playwright/ai-fixture.ts

@@ -108,7 +108,13 @@ export const PlaywrightAiFixture = (options?: {
       | 'aiNumber'
       | 'aiString'
       | 'aiBoolean'
-      | 'aiAsk';
+      | 'aiAsk'
+      | 'runYaml'
+      | 'setAIActionContext'
+      | 'evaluateJavaScript'
+      | 'logScreenshot'
+      | 'freezePageContext'
+      | 'unfreezePageContext';
   }) {
     const { page, testInfo, use, aiActionType } = options;
     const agent = createOrReuseAgentForPage(page, testInfo, {
@@ -373,6 +379,78 @@ export const PlaywrightAiFixture = (options?: {
         aiActionType: 'aiAsk',
       });
     },
+    runYaml: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'runYaml',
+      });
+    },
+    setAIActionContext: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'setAIActionContext',
+      });
+    },
+    evaluateJavaScript: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'evaluateJavaScript',
+      });
+    },
+    logScreenshot: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'logScreenshot',
+      });
+    },
+    freezePageContext: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'freezePageContext',
+      });
+    },
+    unfreezePageContext: async (
+      { page }: { page: OriginPlaywrightPage },
+      use: any,
+      testInfo: TestInfo,
+    ) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: 'unfreezePageContext',
+      });
+    },
   };
 };
 
@@ -420,4 +498,22 @@ export type PlayWrightAiFixtureType = {
   aiAsk: (
     ...args: Parameters<PageAgent['aiAsk']>
   ) => ReturnType<PageAgent['aiAsk']>;
+  runYaml: (
+    ...args: Parameters<PageAgent['runYaml']>
+  ) => ReturnType<PageAgent['runYaml']>;
+  setAIActionContext: (
+    ...args: Parameters<PageAgent['setAIActionContext']>
+  ) => ReturnType<PageAgent['setAIActionContext']>;
+  evaluateJavaScript: (
+    ...args: Parameters<PageAgent['evaluateJavaScript']>
+  ) => ReturnType<PageAgent['evaluateJavaScript']>;
+  logScreenshot: (
+    ...args: Parameters<PageAgent['logScreenshot']>
+  ) => ReturnType<PageAgent['logScreenshot']>;
+  freezePageContext: (
+    ...args: Parameters<PageAgent['freezePageContext']>
+  ) => ReturnType<PageAgent['freezePageContext']>;
+  unfreezePageContext: (
+    ...args: Parameters<PageAgent['unfreezePageContext']>
+  ) => ReturnType<PageAgent['unfreezePageContext']>;
 };