feat(web-integration): reusing context param (#978)

* feat(web-integration): support force context para

* feat(web-integration): update getElementXpath to use UIContext<BaseElement>

* refactor(web-integration): support free context

* refactor(web-integration): support freeze for all VQA api

* feat(site): add freezePageContext and unfreezePageContext methods for improved performance

* refactor(web-integration): remove redundant tests and streamline context handling in freeze/unfreeze functionality

* docs(core): docs for context-freezing (#1013)

* refactor(web-integration): remove isPageContextFrozen flag and simplify context management in PageAgent

---------

Co-authored-by: yuyutaotao <[email protected]>

Author: quanru

apps/chrome-extension/src/scripts/worker.ts

@@ -1,6 +1,6 @@
 /// <reference types="chrome" />
 
-import type { WebUIContext } from '@midscene/web/utils';
+import type { WebUIContext } from '@midscene/web';
 
 const workerMessageTypes = {
   SAVE_CONTEXT: 'save-context',

apps/report/src/components/detail-panel.tsx

@@ -13,7 +13,8 @@ import type {
 } from '@midscene/core';
 import { filterBase64Value, timeStr } from '@midscene/visualizer';
 import { Blackboard, Player } from '@midscene/visualizer';
-import { Segmented } from 'antd';
+import type { WebUIContext } from '@midscene/web';
+import { Segmented, Tooltip } from 'antd';
 import { useEffect, useState } from 'react';
 import OpenInPlayground from './open-in-playground';
 
@@ -49,6 +50,11 @@ const DetailPanel = (): JSX.Element => {
   const imageWidth = useExecutionDump((store) => store.insightWidth);
   const imageHeight = useExecutionDump((store) => store.insightHeight);
 
+  // Check if page context is frozen
+  const isPageContextFrozen = Boolean(
+    (activeTask?.pageContext as WebUIContext)?._isFrozen,
+  );
+
   let availableViewTypes = [VIEW_TYPE_SCREENSHOT, VIEW_TYPE_JSON];
   if (blackboardViewAvailable) {
     availableViewTypes = [
@@ -163,7 +169,11 @@ const DetailPanel = (): JSX.Element => {
     }
     if (type === VIEW_TYPE_BLACKBOARD) {
       return {
-        label: 'Insight',
+        label: isPageContextFrozen ? (
+          <Tooltip title="Current pageContext is frozen">Insight 🧊</Tooltip>
+        ) : (
+          'Insight'
+        ),
         value: type,
         icon: <ScheduleOutlined />,
       };

apps/report/src/components/detail-side.tsx

@@ -226,6 +226,7 @@ const DetailSide = (): JSX.Element => {
   let taskInput: JSX.Element | null = null;
   if (task?.type === 'Planning') {
     const planningTask = task as ExecutionTaskPlanning;
+    const isPageContextFrozen = Boolean((task?.pageContext as any)?._isFrozen);
     if (planningTask.param?.userInstruction) {
       taskInput = MetaKV({
         data: [
@@ -238,6 +239,14 @@ const DetailSide = (): JSX.Element => {
             key: 'log',
             content: planningTask.param.log,
           },
+          ...(isPageContextFrozen
+            ? [
+                {
+                  key: 'context',
+                  content: <Tag color="blue">Frozen Context 🧊</Tag>,
+                },
+              ]
+            : []),
         ],
       });
     } else {
@@ -248,10 +257,19 @@ const DetailSide = (): JSX.Element => {
             key: 'userPrompt',
             content: paramStr(task) || '',
           },
+          ...(isPageContextFrozen
+            ? [
+                {
+                  key: 'context',
+                  content: <Tag color="blue">Frozen Context 🧊</Tag>,
+                },
+              ]
+            : []),
         ],
       });
     }
   } else if (task?.type === 'Insight') {
+    const isPageContextFrozen = Boolean((task?.pageContext as any)?._isFrozen);
     taskInput = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
@@ -279,6 +297,14 @@ const DetailSide = (): JSX.Element => {
               },
             ]
           : []),
+        ...(isPageContextFrozen
+          ? [
+              {
+                key: 'context',
+                content: <Tag color="blue">Frozen Context 🧊</Tag>,
+              },
+            ]
+          : []),
       ],
     });
   } else if (task?.type === 'Action') {

apps/report/src/components/open-in-playground.tsx

@@ -5,7 +5,7 @@ import {
   useEnvConfig,
   useStaticPageAgent,
 } from '@midscene/visualizer';
-import type { WebUIContext } from '@midscene/web/utils';
+import type { WebUIContext } from '@midscene/web';
 import {
   Button,
   ConfigProvider,

apps/site/docs/en/api.mdx

@@ -722,27 +722,61 @@ await agent.logScreenshot('Login page', {
 });
 ```
 
-### `agent._unstableLogContent()`
+### `agent.freezePageContext()`
 
-Retrieve the log content in JSON format from the report file. The structure of the log content may change in the future.
+Freeze the current page context, allowing all subsequent operations to reuse the same page snapshot without retrieving the page state repeatedly. This significantly improves performance when executing a large number of concurrent operations. 
+
+Some notes:
+* Usually, you do not need to use this method, unless you are certain that "context retrieval" is the bottleneck of your test script.
+* You need to call `agent.unfreezePageContext()` in time to restore the real-time page state.
+* Do not call this method in interaction operations, it will make the AI model unable to perceive the latest page state, causing confusing errors.
 
 - Type
 
 ```typescript
-function _unstableLogContent(): Object;
+function freezePageContext(): Promise<void>;
 ```
 
 - Return Value:
 
-  - Returns an object containing the log content.
+  - `Promise<void>`
 
 - Examples:
 
 ```typescript
-const logContent = agent._unstableLogContent();
-console.log(logContent);
+// Freeze the page context
+await agent.freezePageContext();
+
+// Some queries...
+const results = await Promise.all([
+  await agent.aiQuery('Username input box value'),
+  await agent.aiQuery('Password input box value'),
+  await agent.aiLocate('Login button'),
+]);
+console.log(results);
+
+// Unfreeze the page context, subsequent operations will use real-time page state
+await agent.unfreezePageContext();
+```
+
+:::tip
+In the report, operations using frozen context will display a 🧊 icon in the Insight tab.
+:::
+
+### `agent.unfreezePageContext()`
+
+Unfreezes the page context, restoring the use of real-time page state.
+
+- Type
+
+```typescript
+function unfreezePageContext(): Promise<void>;
 ```
 
+- Return Value:
+
+  - `Promise<void>`
+
 ## Properties
 
 ### `.reportFile`

apps/site/docs/zh/api.mdx

@@ -714,6 +714,61 @@ await agent.logScreenshot('登录页面', {
 });
 ```
 
+### `agent.freezePageContext()`
+
+冻结当前页面上下文，使后续所有的操作都复用同一个页面快照，避免多次重复获取页面状态。在执行大量并发操作时，它可以显著提升性能。
+
+一些注意点：
+* 通常情况下，你不需要使用这个方法，除非你确定“页面状态获取”是脚本性能瓶颈。
+* 需要及时调用 `agent.unfreezePageContext()` 来恢复实时页面状态。
+* 不要在交互类操作中使用这个方法，它会让 AI 模型无法感知到页面的最新状态，产生令人困惑的错误。
+
+- 类型
+
+```typescript
+function freezePageContext(): Promise<void>;
+```
+
+- 返回值：
+
+  - `Promise<void>`
+
+- 示例：
+
+```typescript
+// 冻结页面上下文，确保多个操作看到相同的页面状态
+await agent.freezePageContext();
+
+// 执行一些操作...
+const results = await Promise.all([
+  await agent.aiQuery('Username input box value'),
+  await agent.aiQuery('Password input box value'),
+  await agent.aiLocate('Login button'),
+]);
+console.log(results);
+
+// 解冻页面上下文
+await agent.unfreezePageContext();
+```
+
+:::tip
+在报告中，使用冻结上下文的操作会在 Insight tab 中显示 🧊 图标。
+:::
+
+### `agent.unfreezePageContext()`
+
+解冻页面上下文，恢复使用实时的页面状态。
+
+- 类型
+
+```typescript
+function unfreezePageContext(): Promise<void>;
+```
+
+- 返回值：
+
+  - `Promise<void>`
+
 ### `agent._unstableLogContent()`
 
 从报告文件中获取日志内容。日志内容的结构可能会在未来发生变化。

packages/core/src/index.ts

@@ -21,4 +21,6 @@ export type {
   MidsceneYamlFlowItem,
   MidsceneYamlFlowItemAIRightClick,
   MidsceneYamlConfigResult,
+  LocateOption,
+  DetailedLocateParam,
 } from './yaml';

packages/core/src/yaml.ts

@@ -1,9 +1,11 @@
 import type { PlanningActionParamScroll, Rect, TUserPrompt } from './types';
+import type { BaseElement, UIContext } from './types';
 
 export interface LocateOption {
   deepThink?: boolean; // only available in vl model
   cacheable?: boolean; // user can set this param to false to disable the cache for a single agent api
   xpath?: string; // only available in web
+  pageContext?: UIContext<BaseElement>;
 }
 
 export interface InsightExtractOption {

packages/visualizer/src/component/describer.tsx

@@ -5,7 +5,7 @@ import type {
   Rect,
   UIContext,
 } from '@midscene/core';
-import type { WebUIContext } from '@midscene/web/utils';
+import type { WebUIContext } from '@midscene/web';
 import { useEffect, useRef, useState } from 'react';
 import { useStaticPageAgent } from './playground/useStaticPageAgent';
 import './describer.less';

packages/visualizer/src/component/playground/playground-types.ts

@@ -1,7 +1,7 @@
 import type { GroupedActionDump, UIContext } from '@midscene/core';
+import type { WebUIContext } from '@midscene/web';
 import type { ChromeExtensionProxyPageAgent } from '@midscene/web/chrome-extension';
 import type { StaticPageAgent } from '@midscene/web/playground';
-import type { WebUIContext } from '@midscene/web/utils';
 
 // result type
 export interface PlaygroundResult {