chore(core): update implementation for actions

Author: yuyutaotao

packages/android/src/page/index.ts

@@ -18,7 +18,7 @@ import { repeat } from '@midscene/shared/utils';
 import {
   type AndroidDeviceInputOpt,
   type AndroidDevicePage,
-  commonWebActions,
+  commonWebActionsForWebPage,
 } from '@midscene/web';
 import { ADB } from 'appium-adb';
 
@@ -35,51 +35,6 @@ export type AndroidDeviceOpt = {
   imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
 } & AndroidDeviceInputOpt;
 
-const asyncNoop = async () => {};
-const androidActions: DeviceAction[] = [
-  {
-    name: 'AndroidBackButton',
-    description: 'Trigger the system "back" operation on Android devices',
-    location: false,
-    call: asyncNoop,
-  },
-  {
-    name: 'AndroidHomeButton',
-    description: 'Trigger the system "home" operation on Android devices',
-    location: false,
-    call: asyncNoop,
-  },
-  {
-    name: 'AndroidRecentAppsButton',
-    description:
-      'Trigger the system "recent apps" operation on Android devices',
-    location: false,
-    call: asyncNoop,
-  },
-  {
-    name: 'AndroidLongPress',
-    description:
-      'Trigger a long press on the screen at specified coordinates on Android devices',
-    paramSchema: '{ duration?: number }',
-    paramDescription: 'The duration of the long press',
-    location: 'optional',
-    whatToLocate: 'The element to be long pressed',
-    call: asyncNoop,
-  },
-  {
-    name: 'AndroidPull',
-    description:
-      'Trigger pull down to refresh or pull up actions on Android devices',
-    paramSchema:
-      '{ direction: "up" | "down", distance?: number, duration?: number }',
-    paramDescription:
-      'The direction to pull, the distance to pull, and the duration of the pull.',
-    location: 'optional',
-    whatToLocate: 'The element to be pulled',
-    call: asyncNoop,
-  },
-];
-
 export class AndroidDevice implements AndroidDevicePage {
   private deviceId: string;
   private yadbPushed = false;
@@ -92,7 +47,102 @@ export class AndroidDevice implements AndroidDevicePage {
   options?: AndroidDeviceOpt;
 
   actionSpace(): DeviceAction[] {
-    return commonWebActions.concat(androidActions);
+    const commonActions = commonWebActionsForWebPage(this);
+    commonActions.forEach((action) => {
+      if (action.name === 'Input') {
+        action.call = async (context, param) => {
+          const { element } = context;
+          if (element) {
+            await this.clearInput(element as unknown as ElementInfo);
+
+            if (!param || !param.value) {
+              return;
+            }
+          }
+
+          await this.keyboard.type(param.value, {
+            autoDismissKeyboard: this.options?.autoDismissKeyboard,
+          });
+        };
+      }
+    });
+
+    const allActions: DeviceAction[] = [
+      ...commonWebActionsForWebPage(this),
+      {
+        name: 'AndroidBackButton',
+        description: 'Trigger the system "back" operation on Android devices',
+        location: false,
+        call: async (context, param) => {
+          await this.back();
+        },
+      },
+      {
+        name: 'AndroidHomeButton',
+        description: 'Trigger the system "home" operation on Android devices',
+        location: false,
+        call: async (context, param) => {
+          await this.home();
+        },
+      },
+      {
+        name: 'AndroidRecentAppsButton',
+        description:
+          'Trigger the system "recent apps" operation on Android devices',
+        location: false,
+        call: async (context, param) => {
+          await this.recentApps();
+        },
+      },
+      {
+        name: 'AndroidLongPress',
+        description:
+          'Trigger a long press on the screen at specified coordinates on Android devices',
+        paramSchema: '{ duration?: number }',
+        paramDescription: 'The duration of the long press in milliseconds',
+        location: 'required',
+        whatToLocate: 'The element to be long pressed',
+        call: async (context, param) => {
+          const { element } = context;
+          if (!element) {
+            throw new Error(
+              'AndroidLongPress requires an element to be located',
+            );
+          }
+          const [x, y] = element.center;
+          await this.longPress(x, y, param.duration);
+        },
+      } as DeviceAction<{ duration?: number }>,
+      {
+        name: 'AndroidPull',
+        description:
+          'Trigger pull down to refresh or pull up actions on Android devices',
+        paramSchema:
+          '{ direction: "up" | "down", distance?: number, duration?: number }',
+        paramDescription:
+          'The direction to pull, the distance to pull (in pixels), and the duration of the pull (in milliseconds).',
+        location: 'optional',
+        whatToLocate: 'The element to be pulled',
+        call: async (context, param) => {
+          const { element } = context;
+          const startPoint = element
+            ? { left: element.center[0], top: element.center[1] }
+            : undefined;
+          if (param.direction === 'down') {
+            await this.pullDown(startPoint, param.distance, param.duration);
+          } else if (param.direction === 'up') {
+            await this.pullUp(startPoint, param.distance, param.duration);
+          } else {
+            throw new Error(`Unknown pull direction: ${param.direction}`);
+          }
+        },
+      } as DeviceAction<{
+        direction: 'up' | 'down';
+        distance?: number;
+        duration?: number;
+      }>,
+    ];
+    return allActions;
   }
 
   constructor(deviceId: string, options?: AndroidDeviceOpt) {

packages/core/src/ai-model/common.ts

@@ -5,9 +5,9 @@ import type {
   MidsceneYamlFlowItem,
   PlanningAction,
   PlanningActionParamInputOrKeyPress,
-  PlanningActionParamScroll,
   PlanningActionParamSleep,
   Rect,
+  ScrollParam,
   Size,
 } from '@/types';
 import { assert } from '@midscene/shared/utils';
@@ -355,7 +355,7 @@ export function buildYamlFlowFromPlans(
         locate,
       });
     } else if (type === 'Scroll') {
-      const param = plan.param as PlanningActionParamScroll;
+      const param = plan.param as ScrollParam;
       flow.push({
         aiScroll: null,
         locate,

packages/core/src/types.ts

@@ -8,11 +8,7 @@ import type {
   Size,
 } from '@midscene/shared/types';
 import type { ChatCompletionMessageParam } from 'openai/resources';
-import type {
-  DetailedLocateParam,
-  MidsceneYamlFlowItem,
-  scrollParam,
-} from './yaml';
+import type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';
 
 export type {
   ElementTreeNode,
@@ -295,13 +291,12 @@ export interface PlanningAIResponse {
 export type PlanningActionParamTap = null;
 export type PlanningActionParamHover = null;
 export type PlanningActionParamRightClick = null;
+
 export interface PlanningActionParamInputOrKeyPress {
   value: string;
   autoDismissKeyboard?: boolean;
 }
 
-export type PlanningActionParamScroll = scrollParam;
-
 export interface PlanningActionParamAssert {
   assertion: TUserPrompt;
 }
@@ -318,15 +313,12 @@ export type PlanningActionParamWaitFor = AgentWaitForOpt & {
   assertion: string;
 };
 
-export interface PlanningActionParamAndroidLongPress {
-  x: number;
-  y: number;
+export interface AndroidLongPressParam {
   duration?: number;
 }
 
-export interface PlanningActionParamAndroidPull {
+export interface AndroidPullParam {
   direction: 'up' | 'down';
-  startPoint?: { x: number; y: number };
   distance?: number;
   duration?: number;
 }
@@ -623,5 +615,5 @@ export interface DeviceAction<ParamType = any> {
   paramDescription?: string;
   location?: 'required' | 'optional' | false;
   whatToLocate?: string; // what to locate if location is required or optional
-  call: (param: ParamType) => Promise<void> | void;
+  call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;
 }

packages/core/src/yaml.ts

@@ -1,4 +1,4 @@
-import type { PlanningActionParamScroll, Rect, TUserPrompt } from './types';
+import type { Rect, TUserPrompt } from './types';
 import type { BaseElement, UIContext } from './types';
 
 export interface LocateOption {
@@ -24,7 +24,7 @@ export interface DetailedLocateParam extends LocateOption {
   referenceImage?: ReferenceImage;
 }
 
-export interface scrollParam {
+export interface ScrollParam {
   direction: 'down' | 'up' | 'right' | 'left';
   scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
   distance?: null | number; // distance in px
@@ -157,7 +157,7 @@ export interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
 
 export interface MidsceneYamlFlowItemAIScroll
   extends LocateOption,
-    PlanningActionParamScroll {
+    ScrollParam {
   aiScroll: null;
   locate?: TUserPrompt; // which area to scroll, optional
 }

packages/web-integration/src/chrome-extension/page.ts

@@ -7,7 +7,11 @@
 
 import type { WebKeyInput } from '@/common/page';
 import { limitOpenNewTabScript } from '@/common/ui-utils';
-import { type AbstractPage, type MouseButton, commonWebActions } from '@/page';
+import {
+  type AbstractPage,
+  type MouseButton,
+  commonWebActionsForWebPage,
+} from '@/page';
 import type {
   DeviceAction,
   ElementTreeNode,
@@ -56,7 +60,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
   }
 
   actionSpace(): DeviceAction[] {
-    return commonWebActions;
+    return commonWebActionsForWebPage(this);
   }
 
   public async setActiveTabId(tabId: number) {

packages/web-integration/src/common/agent.ts

@@ -20,8 +20,8 @@ import {
   type LocatorValidatorOption,
   type MidsceneYamlScript,
   type OnTaskStartTip,
-  type PlanningActionParamScroll,
   type Rect,
+  type ScrollParam,
   type TUserPrompt,
 } from '@midscene/core';
 
@@ -411,7 +411,7 @@ export class PageAgent<PageType extends WebPage = WebPage> {
   }
 
   async aiScroll(
-    scrollParam: PlanningActionParamScroll,
+    scrollParam: ScrollParam,
     locatePrompt?: TUserPrompt,
     opt?: LocateOption,
   ) {

packages/web-integration/src/common/page.d.ts

@@ -49,7 +49,7 @@ export interface AndroidDevicePage extends AbstractPage {
 
 export type AndroidDeviceInputOpt = {
   autoDismissKeyboard?: boolean;
-  hideKeyboardStrategy?: 'esc-first' | 'back-first';
+  keyboardDismissStrategy?: 'esc-first' | 'back-first';
 };
 
 export type WebPage =