feat(core): use action space to expose actions (#1031)

* chore(core): update prompt

* feat(core): define action space for llm planning

* chore(core): use action space to declare actions for agent

* fix(core): prompt for planning

* fix(core): lint

* feat(core): update prompt for llm planning

* fix(core): prompt for planning

Author: yuyutaotao

packages/android/src/page/index.ts

@@ -3,7 +3,7 @@ import { randomUUID } from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { type Point, type Size, getAIConfig } from '@midscene/core';
-import type { PageType } from '@midscene/core';
+import type { DeviceAction, PageType } from '@midscene/core';
 import { getTmpFile, sleep } from '@midscene/core/utils';
 import {
   MIDSCENE_ADB_PATH,
@@ -15,7 +15,11 @@ import type { ElementInfo } from '@midscene/shared/extractor';
 import { isValidPNGImageBuffer, resizeImg } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import { repeat } from '@midscene/shared/utils';
-import type { AndroidDeviceInputOpt, AndroidDevicePage } from '@midscene/web';
+import {
+  type AndroidDeviceInputOpt,
+  type AndroidDevicePage,
+  commonWebActions,
+} from '@midscene/web';
 import { ADB } from 'appium-adb';
 
 // only for Android, because it's impossible to scroll to the bottom, so we need to set a default scroll times
@@ -31,6 +35,51 @@ export type AndroidDeviceOpt = {
   imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
 } & AndroidDeviceInputOpt;
 
+const asyncNoop = async () => {};
+const androidActions: DeviceAction[] = [
+  {
+    name: 'AndroidBackButton',
+    description: 'Trigger the system "back" operation on Android devices',
+    location: false,
+    call: asyncNoop,
+  },
+  {
+    name: 'AndroidHomeButton',
+    description: 'Trigger the system "home" operation on Android devices',
+    location: false,
+    call: asyncNoop,
+  },
+  {
+    name: 'AndroidRecentAppsButton',
+    description:
+      'Trigger the system "recent apps" operation on Android devices',
+    location: false,
+    call: asyncNoop,
+  },
+  {
+    name: 'AndroidLongPress',
+    description:
+      'Trigger a long press on the screen at specified coordinates on Android devices',
+    paramSchema: '{ duration?: number }',
+    paramDescription: 'The duration of the long press',
+    location: 'optional',
+    whatToLocate: 'The element to be long pressed',
+    call: asyncNoop,
+  },
+  {
+    name: 'AndroidPull',
+    description:
+      'Trigger pull down to refresh or pull up actions on Android devices',
+    paramSchema:
+      '{ direction: "up" | "down", distance?: number, duration?: number }',
+    paramDescription:
+      'The direction to pull, the distance to pull, and the duration of the pull.',
+    location: 'optional',
+    whatToLocate: 'The element to be pulled',
+    call: asyncNoop,
+  },
+];
+
 export class AndroidDevice implements AndroidDevicePage {
   private deviceId: string;
   private yadbPushed = false;
@@ -42,6 +91,10 @@ export class AndroidDevice implements AndroidDevicePage {
   uri: string | undefined;
   options?: AndroidDeviceOpt;
 
+  actionSpace(): DeviceAction[] {
+    return commonWebActions.concat(androidActions);
+  }
+
   constructor(deviceId: string, options?: AndroidDeviceOpt) {
     assert(deviceId, 'deviceId is required for AndroidDevice');
 

packages/core/src/ai-model/common.ts

@@ -378,7 +378,6 @@ export function buildYamlFlowFromPlans(
       // not implemented in yaml yet
     } else if (
       type === 'Error' ||
-      type === 'ExpectedFalsyCondition' ||
       type === 'Assert' ||
       type === 'AssertWithoutThrow' ||
       type === 'Finished'

packages/core/src/ai-model/llm-planning.ts

@@ -1,4 +1,9 @@
-import type { PageType, PlanningAIResponse, UIContext } from '@/types';
+import type {
+  DeviceAction,
+  PageType,
+  PlanningAIResponse,
+  UIContext,
+} from '@/types';
 import { vlLocateMode } from '@midscene/shared/env';
 import { paddingToMatchBlockByBase64 } from '@midscene/shared/img';
 import { assert } from '@midscene/shared/utils';
@@ -23,6 +28,7 @@ export async function plan(
   opts: {
     context: UIContext;
     pageType: PageType;
+    actionSpace: DeviceAction[];
     callAI?: typeof callAiFn<PlanningAIResponse>;
     log?: string;
     actionContext?: string;
@@ -34,7 +40,7 @@ export async function plan(
     await describeUserPage(context);
 
   const systemPrompt = await systemPromptToTaskPlanning({
-    pageType: opts.pageType,
+    actionSpace: opts.actionSpace,
     vlMode: vlLocateMode(),
   });
   const taskBackgroundContextText = generateTaskBackgroundContext(

packages/core/src/ai-model/prompt/llm-planning.ts

@@ -266,7 +266,6 @@ export interface PlanningAction<ParamType = any> {
     | 'KeyboardPress'
     | 'Scroll'
     | 'Error'
-    | 'ExpectedFalsyCondition'
     | 'Assert'
     | 'AssertWithoutThrow'
     | 'Sleep'
@@ -293,12 +292,6 @@ export interface PlanningAIResponse {
   yamlString?: string;
 }
 
-// export interface PlanningFurtherPlan {
-//   whatToDoNext: string;
-//   log: string;
-// }
-// export type PlanningActionParamPlan = PlanningFurtherPlan;
-
 export type PlanningActionParamTap = null;
 export type PlanningActionParamHover = null;
 export type PlanningActionParamRightClick = null;
@@ -622,3 +615,13 @@ export type TUserPrompt =
   | ({
       prompt: string;
     } & Partial<TMultimodalPrompt>);
+
+export interface DeviceAction<ParamType = any> {
+  name: string;
+  description?: string;
+  paramSchema?: string;
+  paramDescription?: string;
+  location?: 'required' | 'optional' | false;
+  whatToLocate?: string; // what to locate if location is required or optional
+  call: (param: ParamType) => Promise<void> | void;
+}

packages/core/src/types.ts

@@ -1,5 +1,6 @@
 import { plan } from '@/ai-model';
 import { vlLocateMode } from '@midscene/shared/env';
+import { mockActionSpace } from 'tests/common';
 import { getContextFromFixture } from 'tests/evaluation';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
@@ -19,9 +20,12 @@ describe.skipIf(vlMode)('automation - llm planning', () => {
       'type "Why is the earth a sphere?", wait 3.5s, hit Enter',
       {
         context,
+        actionSpace: mockActionSpace,
+        pageType: 'puppeteer',
       },
     );
     expect(actions).toBeTruthy();
+
     expect(actions!.length).toBe(3);
     expect(actions![0].type).toBe('Input');
     expect(actions![1].type).toBe('Sleep');
@@ -34,7 +38,7 @@ describe.skipIf(vlMode)('automation - llm planning', () => {
     const { context } = await getContextFromFixture('todo');
     const { actions } = await plan(
       'Scroll down the page by 200px, scroll up the page by 100px, scroll right the second item of the task list by 300px',
-      { context },
+      { context, actionSpace: mockActionSpace, pageType: 'puppeteer' },
     );
     expect(actions).toBeTruthy();
     expect(actions!.length).toBe(3);
@@ -79,7 +83,11 @@ describe('planning', () => {
   todoInstructions.forEach(({ name, instruction }) => {
     it(`todo mvc - ${name}`, async () => {
       const { context } = await getContextFromFixture('todo');
-      const { actions } = await plan(instruction, { context });
+      const { actions } = await plan(instruction, {
+        context,
+        actionSpace: mockActionSpace,
+        pageType: 'puppeteer',
+      });
       expect(actions).toBeTruthy();
       expect(actions![0].locate).toBeTruthy();
       expect(actions![0].locate?.prompt).toBeTruthy();
@@ -93,29 +101,31 @@ describe('planning', () => {
       'Scroll left the status filters (with a button named "completed")',
       {
         context,
+        actionSpace: mockActionSpace,
+        pageType: 'puppeteer',
       },
     );
     expect(actions).toBeTruthy();
     expect(actions![0].type).toBe('Scroll');
     expect(actions![0].locate).toBeTruthy();
   });
 
-  it.skip('should not throw in an "if" statement', async () => {
+  it('should not throw in an "if" statement', async () => {
     const { context } = await getContextFromFixture('todo');
     const { actions, error } = await plan(
       'If there is a cookie prompt, close it',
-      { context },
+      { context, actionSpace: mockActionSpace, pageType: 'puppeteer' },
     );
 
-    expect(actions?.length === 1).toBeTruthy();
-    expect(actions?.[0]!.type).toBe('ExpectedFalsyCondition');
+    expect(error).toBeFalsy();
+    expect(actions?.length).toBe(0);
   });
 
   it('should make mark unfinished when something is not found', async () => {
     const { context } = await getContextFromFixture('todo');
     const res = await plan(
-      'click the input box, wait 300ms, click the close button of the cookie prompt',
-      { context },
+      'click the input box, wait 300ms. After that, the page will be redirected to the home page, click the close button of the cookie prompt on the home page',
+      { context, actionSpace: mockActionSpace, pageType: 'puppeteer' },
     );
 
     expect(res.more_actions_needed_by_instruction).toBeTruthy();

packages/core/tests/ai/llm-planning/basic.test.ts

@@ -0,0 +1,45 @@
+import type { DeviceAction } from '@/types';
+
+export const mockActionSpace: DeviceAction[] = [
+  {
+    name: 'Tap',
+    description: 'Tap the element',
+    location: 'required',
+    whatToLocate: 'The element to be tapped',
+    paramSchema: '{ value: string }',
+    paramDescription: 'The value to be tapped',
+    call: async () => {},
+  },
+  {
+    name: 'Sleep',
+    description: 'Sleep for a period of time',
+    paramSchema: '{ timeMs: number }',
+    paramDescription: 'The duration of the sleep in milliseconds',
+    location: false,
+    call: async () => {},
+  },
+  {
+    name: 'Input',
+    description: 'Input text into the input field',
+    paramSchema: '{ value: string }',
+    paramDescription: 'The value to be input',
+    location: 'optional',
+    call: async () => {},
+  },
+  {
+    name: 'KeyboardPress',
+    description: 'Press a keyboard key',
+    paramSchema: '{ value: string }',
+    paramDescription: 'The value to be input',
+    location: 'optional',
+    call: async () => {},
+  },
+  {
+    name: 'Scroll',
+    description: 'Scroll the page',
+    paramSchema: '{ value: string }',
+    paramDescription: 'The value to be input',
+    location: 'optional',
+    call: async () => {},
+  },
+];