feat(web-integration): add executeAction method to various classes and update action handling logic (#1059)

* feat(web-integration): add executeAction method to various classes and update action handling logic

* feat(web-integration): refactor action handling by introducing executeActionForPage utility across multiple classes

Author: quanru

packages/android/src/page/index.ts

@@ -4,7 +4,7 @@ import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import path from 'node:path';
 import { type Point, type Size, getAIConfig } from '@midscene/core';
-import type { DeviceAction, PageType } from '@midscene/core';
+import type { DeviceAction, ExecutorContext, PageType } from '@midscene/core';
 import { getTmpFile, sleep } from '@midscene/core/utils';
 import {
   MIDSCENE_ADB_PATH,
@@ -17,7 +17,10 @@ import { isValidPNGImageBuffer, resizeImg } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import { repeat } from '@midscene/shared/utils';
 import type { AndroidDeviceInputOpt, AndroidDevicePage } from '@midscene/web';
-import { commonWebActionsForWebPage } from '@midscene/web/utils';
+import {
+  commonWebActionsForWebPage,
+  executeActionForPage,
+} from '@midscene/web/utils';
 
 import { ADB } from 'appium-adb';
 
@@ -147,6 +150,14 @@ export class AndroidDevice implements AndroidDevicePage {
     return allActions;
   }
 
+  async executeAction<T = unknown>(
+    actionName: string,
+    context: ExecutorContext,
+    param: T,
+  ): Promise<void> {
+    return executeActionForPage(this, actionName, context, param);
+  }
+
   constructor(deviceId: string, options?: AndroidDeviceOpt) {
     assert(deviceId, 'deviceId is required for AndroidDevice');
 

packages/web-integration/src/bridge-mode/agent-cli-side.ts

@@ -1,5 +1,6 @@
 import { PageAgent, type PageAgentOpt } from '@/common/agent';
 import type { KeyboardAction, MouseAction } from '@/page';
+import type { ExecutorContext } from '@midscene/core';
 import { assert } from '@midscene/shared/utils';
 import {
   type BridgeConnectTabOptions,
@@ -14,6 +15,11 @@ import type { ExtensionBridgePageBrowserSide } from './page-browser-side';
 
 interface ChromeExtensionPageCliSide extends ExtensionBridgePageBrowserSide {
   showStatusMessage: (message: string) => Promise<void>;
+  executeAction: <T = unknown>(
+    actionName: string,
+    context: ExecutorContext,
+    param: T,
+  ) => Promise<void>;
 }
 
 const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -42,6 +48,17 @@ export const getBridgePageInCliSide = (
     showStatusMessage: async (message: string) => {
       await server.call(BridgeEvent.UpdateAgentStatus, [message]);
     },
+    executeAction: async <T = unknown>(
+      actionName: string,
+      context: ExecutorContext,
+      param: T,
+    ): Promise<void> => {
+      return await server.call(BridgeEvent.ExecuteAction, [
+        actionName,
+        context,
+        param,
+      ]);
+    },
   };
 
   return new Proxy(page, {

packages/web-integration/src/bridge-mode/common.ts

@@ -14,6 +14,7 @@ export enum BridgeEvent {
   GetBrowserTabList = 'getBrowserTabList',
   SetDestroyOptions = 'setDestroyOptions',
   SetActiveTabId = 'setActiveTabId',
+  ExecuteAction = 'executeAction',
 }
 
 export const BridgeSignalKill = 'MIDSCENE_BRIDGE_SIGNAL_KILL';

packages/web-integration/src/bridge-mode/page-browser-side.ts

@@ -62,6 +62,16 @@ export class ExtensionBridgePageBrowserSide extends ChromeExtensionProxyPage {
           return this.onLogMessage(args[0] as string, 'status');
         }
 
+        if (method === BridgeEvent.ExecuteAction) {
+          const [actionName, context, param] = args;
+          const actionSpace = this.actionSpace();
+          const action = actionSpace.find((a) => a.name === actionName);
+          if (!action) {
+            throw new Error(`Action ${actionName} not found in action space`);
+          }
+          return action.call(context, param);
+        }
+
         const tabId = await this.getActiveTabId();
         if (!tabId || tabId === 0) {
           throw new Error('no tab is connected');

packages/web-integration/src/chrome-extension/page.ts

@@ -7,11 +7,15 @@
 
 import type { WebKeyInput } from '@/common/page';
 import { limitOpenNewTabScript } from '@/common/ui-utils';
-import { commonWebActionsForWebPage } from '@/common/utils';
+import {
+  commonWebActionsForWebPage,
+  executeActionForPage,
+} from '@/common/utils';
 import type { AbstractPage, MouseButton } from '@/page';
 import type {
   DeviceAction,
   ElementTreeNode,
+  ExecutorContext,
   Point,
   Size,
 } from '@midscene/core';
@@ -60,6 +64,14 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
     return commonWebActionsForWebPage(this);
   }
 
+  async executeAction<T = unknown>(
+    actionName: string,
+    context: ExecutorContext,
+    param: T,
+  ): Promise<void> {
+    return executeActionForPage(this, actionName, context, param);
+  }
+
   public async setActiveTabId(tabId: number) {
     if (this.activeTabId) {
       throw new Error(

packages/web-integration/src/common/tasks.ts

@@ -203,15 +203,15 @@ export class PageTaskExecutor {
     },
   ) {
     const tasks: ExecutionTaskApply[] = [];
-    plans.forEach((plan) => {
+    for (const plan of plans) {
       if (plan.type === 'Locate') {
         if (
           plan.locate === null ||
           plan.locate?.id === null ||
           plan.locate?.id === 'null'
         ) {
           // console.warn('Locate action with id is null, will be ignored');
-          return;
+          continue;
         }
         const taskFind: ExecutionTaskInsightLocateApply = {
           type: 'Insight',
@@ -495,13 +495,7 @@ export class PageTaskExecutor {
         };
         tasks.push(taskActionDrag);
       } else {
-        const actionSpace = this.page.actionSpace();
         const planType = plan.type;
-        const action = actionSpace.find((action) => action.name === planType);
-        if (!action) {
-          throw new Error(`Action ${planType} not found in action space`);
-        }
-
         const task: ExecutionTaskActionApply = {
           type: 'Action',
           subType: planType,
@@ -510,17 +504,16 @@ export class PageTaskExecutor {
           executor: async (param, context) => {
             debug(
               'executing action',
-              action.name,
+              planType,
               param,
               `context.element.center: ${context.element?.center}`,
             );
-            return action.call(context, param);
+            return await this.page.executeAction(planType, context, param);
           },
         };
-        assert(task, `Unknown or unsupported task type: ${planType}`);
         tasks.push(task);
       }
-    });
+    }
 
     const wrappedTasks = tasks.map(
       (task: ExecutionTaskApply, index: number) => {

packages/web-integration/src/common/utils.ts

@@ -5,6 +5,7 @@ import type {
   ElementTreeNode,
   ExecutionDump,
   ExecutionTask,
+  ExecutorContext,
   PlanningLocateParam,
   PlaywrightParserOpt,
   ScrollParam,
@@ -348,6 +349,22 @@ export const parsePrompt = (
   };
 };
 
+export const executeActionForPage = async <T extends AbstractPage, P = unknown>(
+  page: T,
+  actionName: string,
+  context: ExecutorContext,
+  param: P,
+): Promise<void> => {
+  const actions = await page.actionSpace();
+  const action = actions.find((a) => a.name === actionName);
+  
+  if (!action) {
+    throw new Error(`Action ${actionName} not found in action space`);
+  }
+  
+  return action.call(context, param);
+};
+
 export const commonWebActionsForWebPage = <T extends AbstractPage>(
   page: T,
 ): DeviceAction[] => [

packages/web-integration/src/page.ts

@@ -1,4 +1,10 @@
-import type { DeviceAction, Point, ScrollParam, Size } from '@midscene/core';
+import type {
+  DeviceAction,
+  ExecutorContext,
+  Point,
+  ScrollParam,
+  Size,
+} from '@midscene/core';
 import { sleep } from '@midscene/core/utils';
 import type { ElementInfo, ElementNode } from '@midscene/shared/extractor';
 import { assert } from '@midscene/shared/utils';
@@ -43,7 +49,12 @@ export abstract class AbstractPage {
   abstract url(): string | Promise<string>;
   abstract screenshotBase64?(): Promise<string>;
   abstract size(): Promise<Size>;
-  abstract actionSpace(): DeviceAction[];
+  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;
+  abstract executeAction<T = unknown>(
+    actionName: string,
+    context: ExecutorContext,
+    param: T,
+  ): Promise<void>;
 
   get mouse(): MouseAction {
     return {
@@ -94,124 +105,3 @@ export abstract class AbstractPage {
 
   abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
 }
-
-export const commonWebActionsForWebPage = <T extends AbstractPage>(
-  page: T,
-): DeviceAction[] => [
-  {
-    name: 'Tap',
-    description: 'Tap the element',
-    location: 'required',
-    call: async (context) => {
-      const { element } = context;
-      assert(element, 'Element not found, cannot tap');
-      await page.mouse.click(element.center[0], element.center[1], {
-        button: 'left',
-      });
-    },
-  },
-  {
-    name: 'RightClick',
-    description: 'Right click the element',
-    location: 'required',
-    call: async (context) => {
-      const { element } = context;
-      assert(element, 'Element not found, cannot right click');
-      await page.mouse.click(element.center[0], element.center[1], {
-        button: 'right',
-      });
-    },
-  },
-  {
-    name: 'Hover',
-    description: 'Move the mouse to the element',
-    location: 'required',
-    call: async (context) => {
-      const { element } = context;
-      assert(element, 'Element not found, cannot hover');
-      await page.mouse.move(element.center[0], element.center[1]);
-    },
-  },
-  {
-    name: 'Input',
-    description: 'Replace the input field with a new value',
-    paramSchema: '{ value: string }',
-    paramDescription:
-      '`value` is the final that should be filled in the input box. No matter what modifications are required, just provide the final value to replace the existing input value. Giving a blank string means clear the input field.',
-    location: 'required',
-    whatToLocate: 'The input field to be filled',
-    call: async (context, param) => {
-      const { element } = context;
-      if (element) {
-        await page.clearInput(element as unknown as ElementInfo);
-
-        if (!param || !param.value) {
-          return;
-        }
-      }
-
-      // Note: there is another implementation in AndroidDevicePage, which is more complex
-      await page.keyboard.type(param.value);
-    },
-  } as DeviceAction<{ value: string }>,
-  {
-    name: 'KeyboardPress',
-    description: 'Press a key',
-    paramSchema: '{ value: string }',
-    paramDescription: 'The key to be pressed',
-    location: false,
-    call: async (context, param) => {
-      const keys = getKeyCommands(param.value);
-      await page.keyboard.press(keys as any); // TODO: fix this type error
-    },
-  } as DeviceAction<{ value: string }>,
-  {
-    name: 'Scroll',
-    description: 'Scroll the page or an element',
-    paramSchema:
-      '{ direction: "down"(default) | "up" | "right" | "left", scrollType: "once" (default) | "untilBottom" | "untilTop" | "untilRight" | "untilLeft", distance: number | null }',
-    paramDescription:
-      'The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',
-    location: 'optional',
-    whatToLocate: 'The element to be scrolled',
-    call: async (context, param) => {
-      const { element } = context;
-      const startingPoint = element
-        ? {
-            left: element.center[0],
-            top: element.center[1],
-          }
-        : undefined;
-      const scrollToEventName = param?.scrollType;
-      if (scrollToEventName === 'untilTop') {
-        await page.scrollUntilTop(startingPoint);
-      } else if (scrollToEventName === 'untilBottom') {
-        await page.scrollUntilBottom(startingPoint);
-      } else if (scrollToEventName === 'untilRight') {
-        await page.scrollUntilRight(startingPoint);
-      } else if (scrollToEventName === 'untilLeft') {
-        await page.scrollUntilLeft(startingPoint);
-      } else if (scrollToEventName === 'once' || !scrollToEventName) {
-        if (param?.direction === 'down' || !param || !param.direction) {
-          await page.scrollDown(param?.distance || undefined, startingPoint);
-        } else if (param.direction === 'up') {
-          await page.scrollUp(param.distance || undefined, startingPoint);
-        } else if (param.direction === 'left') {
-          await page.scrollLeft(param.distance || undefined, startingPoint);
-        } else if (param.direction === 'right') {
-          await page.scrollRight(param.distance || undefined, startingPoint);
-        } else {
-          throw new Error(`Unknown scroll direction: ${param.direction}`);
-        }
-        // until mouse event is done
-        await sleep(500);
-      } else {
-        throw new Error(
-          `Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(
-            param,
-          )}`,
-        );
-      }
-    },
-  } as DeviceAction<ScrollParam>,
-];

packages/web-integration/src/playground/static-page.ts

@@ -1,6 +1,6 @@
-import { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';
+import { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED, executeActionForPage } from '@/common/utils';
 import type { AbstractPage } from '@/page';
-import type { DeviceAction, Point } from '@midscene/core';
+import type { DeviceAction, ExecutorContext, Point } from '@midscene/core';
 import type { WebUIContext } from '../web-element';
 
 const ThrowNotImplemented: any = (methodName: string) => {
@@ -32,6 +32,14 @@ export default class StaticPage implements AbstractPage {
     return [];
   }
 
+  async executeAction<T = unknown>(
+    actionName: string,
+    context: ExecutorContext,
+    param: T,
+  ): Promise<void> {
+    return executeActionForPage(this, actionName, context, param);
+  }
+
   async evaluateJavaScript<T = any>(script: string): Promise<T> {
     return ThrowNotImplemented('evaluateJavaScript');
   }