feat(android): add long press functionality (#963)

* feat(android): add long press functionality

* feat(android): add pull down and pull up functionality for refresh actions

* feat(android): update action log to include AndroidPull for refresh actions

Author: quanru

packages/android/src/page/index.ts

@@ -779,6 +779,73 @@ ${Object.keys(size)
     await adb.shell('input keyevent 187');
   }
 
+  async longPress(x: number, y: number, duration = 1000): Promise<void> {
+    const adb = await this.getAdb();
+
+    // Use adjusted coordinates
+    const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
+    await adb.shell(
+      `input swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`,
+    );
+  }
+
+  async pullDown(
+    startPoint?: Point,
+    distance?: number,
+    duration = 800,
+  ): Promise<void> {
+    const { width, height } = await this.size();
+
+    // Default start point is near top of screen (but not too close to edge)
+    const start = startPoint
+      ? { x: startPoint.left, y: startPoint.top }
+      : { x: width / 2, y: height * 0.15 };
+
+    // Default distance is larger to ensure refresh is triggered
+    const pullDistance = distance || height * 0.5;
+    const end = { x: start.x, y: start.y + pullDistance };
+
+    // Use custom drag with specified duration for better pull-to-refresh detection
+    await this.pullDrag(start, end, duration);
+    await sleep(200); // Give more time for refresh to start
+  }
+
+  private async pullDrag(
+    from: { x: number; y: number },
+    to: { x: number; y: number },
+    duration: number,
+  ): Promise<void> {
+    const adb = await this.getAdb();
+
+    // Use adjusted coordinates
+    const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
+    const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
+
+    // Use the specified duration for better pull gesture recognition
+    await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`);
+  }
+
+  async pullUp(
+    startPoint?: Point,
+    distance?: number,
+    duration = 600,
+  ): Promise<void> {
+    const { width, height } = await this.size();
+
+    // Default start point is bottom center of screen
+    const start = startPoint
+      ? { x: startPoint.left, y: startPoint.top }
+      : { x: width / 2, y: height * 0.85 };
+
+    // Default distance is 1/3 of screen height
+    const pullDistance = distance || height * 0.4;
+    const end = { x: start.x, y: start.y - pullDistance };
+
+    // Use pullDrag for consistent pull gesture handling
+    await this.pullDrag(start, end, duration);
+    await sleep(100);
+  }
+
   async getXpathsById(id: string): Promise<string[]> {
     throw new Error('Not implemented');
   }

packages/core/src/ai-model/common.ts

@@ -376,7 +376,9 @@ export function buildYamlFlowFromPlans(
     } else if (
       type === 'AndroidBackButton' ||
       type === 'AndroidHomeButton' ||
-      type === 'AndroidRecentAppsButton'
+      type === 'AndroidRecentAppsButton' ||
+      type === 'AndroidLongPress' ||
+      type === 'AndroidPull'
     ) {
       // not implemented in yaml yet
     } else if (

packages/core/src/ai-model/prompt/llm-planning.ts

@@ -25,7 +25,7 @@ Target: User will give you a screenshot, an instruction and some previous logs i
 
 Restriction:
 - Don't give extra actions or plans beyond the instruction. ONLY plan for what the instruction requires. For example, don't try to submit the form if the instruction is only to fill something.
-- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === 'android' ? ', AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton.' : '.'}
+- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === 'android' ? ', AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull.' : '.'}
 - Don't repeat actions in the previous logs.
 - Bbox is the bounding box of the element to be located. It's an array of 4 numbers, representing ${bboxDescription(vlMode)}.
 
@@ -40,7 +40,9 @@ ${
   pageType === 'android'
     ? `- AndroidBackButton: { type: "AndroidBackButton", param: {} }
 - AndroidHomeButton: { type: "AndroidHomeButton", param: {} }
-- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }`
+- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }
+- AndroidLongPress: { type: "AndroidLongPress", param: { x: number, y: number, duration?: number } }
+- AndroidPull: { type: "AndroidPull", param: { direction: 'up' | 'down', startPoint?: { x: number, y: number }, distance?: number, duration?: number } } // Pull down to refresh (direction: 'down') or pull up to load more (direction: 'up')`
     : ''
 }
 
@@ -93,7 +95,7 @@ You are a versatile professional in software UI automation. Your outstanding con
 ## Workflow
 
 1. Receive the screenshot, element description of screenshot(if any), user's instruction and previous logs.
-2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === 'android' ? '/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton' : ''}). The "About the action" section below will give you more details.
+2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === 'android' ? '/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton / AndroidLongPress / AndroidPull' : ''}). The "About the action" section below will give you more details.
 3. Precisely locate the target element if it's already shown in the screenshot, put the location info in the \`locate\` field of the action.
 4. If some target elements is not shown in the screenshot, consider the user's instruction is not feasible on this page. Follow the next steps.
 5. Consider whether the user's instruction will be accomplished after all the actions
@@ -154,7 +156,11 @@ ${
 - type: 'AndroidHomeButton', trigger the system "home" operation on Android devices
   * {{ param: {{}} }}
 - type: 'AndroidRecentAppsButton', trigger the system "recent apps" operation on Android devices
-  * {{ param: {{}} }}`
+  * {{ param: {{}} }}
+- type: 'AndroidLongPress', trigger a long press on the screen at specified coordinates on Android devices
+  * {{ param: {{ x: number, y: number, duration?: number }} }}
+- type: 'AndroidPull', trigger pull down to refresh or pull up actions on Android devices
+  * {{ param: {{ direction: 'up' | 'down', startPoint?: {{ x: number, y: number }}, distance?: number, duration?: number }} }}`
     : ''
 }
 `;
@@ -278,7 +284,7 @@ export const planSchema: ResponseFormatJSONSchema = {
               type: {
                 type: 'string',
                 description:
-                  'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton"',
+                  'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton", "AndroidLongPress"',
               },
               param: {
                 anyOf: [

packages/core/src/ai-model/ui-tars-planning.ts

@@ -28,7 +28,9 @@ type ActionType =
   | 'wait'
   | 'androidBackButton'
   | 'androidHomeButton'
-  | 'androidRecentAppsButton';
+  | 'androidRecentAppsButton'
+  | 'androidLongPress'
+  | 'androidPull';
 
 const debug = getDebug('ui-tars-planning');
 const bboxSize = 10;
@@ -199,6 +201,42 @@ export async function vlmPlanning(options: {
         type: 'AndroidRecentAppsButton',
         param: {},
       });
+    } else if (action.action_type === 'androidLongPress') {
+      assert(
+        action.action_inputs.start_coords,
+        'start_coords is required for androidLongPress',
+      );
+      const point = action.action_inputs.start_coords;
+      transformActions.push({
+        type: 'AndroidLongPress',
+        param: {
+          x: point[0],
+          y: point[1],
+          duration: 1000,
+        },
+        locate: null,
+        thought: action.thought || '',
+      });
+    } else if (action.action_type === 'androidPull') {
+      const pullDirection = action.action_inputs.direction || 'down';
+      const startPoint = action.action_inputs.start_coords
+        ? {
+            x: action.action_inputs.start_coords[0],
+            y: action.action_inputs.start_coords[1],
+          }
+        : undefined;
+
+      transformActions.push({
+        type: 'AndroidPull',
+        param: {
+          direction: pullDirection as 'up' | 'down',
+          startPoint,
+          distance: (action.action_inputs as any).distance,
+          duration: (action.action_inputs as any).duration || 500,
+        },
+        locate: null,
+        thought: action.thought || '',
+      });
     }
   });
 
@@ -315,14 +353,23 @@ interface FinishedAction extends BaseAction {
   action_inputs: Record<string, never>;
 }
 
+interface AndroidLongPressAction extends BaseAction {
+  action_type: 'androidLongPress';
+  action_inputs: {
+    start_coords: [number, number]; // Coordinates for long press
+    duration?: number; // Duration in milliseconds
+  };
+}
+
 export type Action =
   | ClickAction
   | DragAction
   | TypeAction
   | HotkeyAction
   | ScrollAction
   | FinishedAction
-  | WaitAction;
+  | WaitAction
+  | AndroidLongPressAction;
 
 export async function resizeImageForUiTars(imageBase64: string, size: Size) {
   if (

packages/core/src/types.ts

@@ -272,7 +272,9 @@ export interface PlanningAction<ParamType = any> {
     | 'Finished'
     | 'AndroidBackButton'
     | 'AndroidHomeButton'
-    | 'AndroidRecentAppsButton';
+    | 'AndroidRecentAppsButton'
+    | 'AndroidLongPress'
+    | 'AndroidPull';
   param: ParamType;
   locate?: PlanningLocateParam | null;
 }
@@ -321,6 +323,19 @@ export interface PlanningActionParamError {
 export type PlanningActionParamWaitFor = AgentWaitForOpt & {
   assertion: string;
 };
+
+export interface PlanningActionParamAndroidLongPress {
+  x: number;
+  y: number;
+  duration?: number;
+}
+
+export interface PlanningActionParamAndroidPull {
+  direction: 'up' | 'down';
+  startPoint?: { x: number; y: number };
+  distance?: number;
+  duration?: number;
+}
 /**
  * misc
  */

packages/core/tests/unit-test/prompt/__snapshots__/prompt.test.ts.snap

@@ -483,7 +483,7 @@ exports[`system prompts > planning - 4o - response format 1`] = `
                 "type": "string",
               },
               "type": {
-                "description": "Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton"",
+                "description": "Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton", "AndroidLongPress"",
                 "type": "string",
               },
             },
@@ -683,7 +683,7 @@ Target: User will give you a screenshot, an instruction and some previous logs i
 
 Restriction:
 - Don't give extra actions or plans beyond the instruction. ONLY plan for what the instruction requires. For example, don't try to submit the form if the instruction is only to fill something.
-- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll, AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton.
+- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll, AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull.
 - Don't repeat actions in the previous logs.
 - Bbox is the bounding box of the element to be located. It's an array of 4 numbers, representing 2d bounding box as [xmin, ymin, xmax, ymax].
 
@@ -697,6 +697,8 @@ Supporting actions:
 - AndroidBackButton: { type: "AndroidBackButton", param: {} }
 - AndroidHomeButton: { type: "AndroidHomeButton", param: {} }
 - AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }
+- AndroidLongPress: { type: "AndroidLongPress", param: { x: number, y: number, duration?: number } }
+- AndroidPull: { type: "AndroidPull", param: { direction: 'up' | 'down', startPoint?: { x: number, y: number }, distance?: number, duration?: number } } // Pull down to refresh (direction: 'down') or pull up to load more (direction: 'up')
 
 Field description:
 * The \`prompt\` field inside the \`locate\` field is a short description that could be used to locate the element.

packages/web-integration/src/common/page.d.ts

@@ -34,6 +34,17 @@ export interface AndroidDevicePage extends AbstractPage {
   back(): Promise<void>;
   home(): Promise<void>;
   recentApps(): Promise<void>;
+  longPress(x: number, y: number, duration?: number): Promise<void>;
+  pullDown(
+    startPoint?: Point,
+    distance?: number,
+    duration?: number,
+  ): Promise<void>;
+  pullUp(
+    startPoint?: Point,
+    distance?: number,
+    duration?: number,
+  ): Promise<void>;
 }
 
 export type AndroidDeviceInputOpt = {

packages/web-integration/src/common/tasks.ts

@@ -24,6 +24,8 @@ import {
   type PageType,
   type PlanningAIResponse,
   type PlanningAction,
+  type PlanningActionParamAndroidLongPress,
+  type PlanningActionParamAndroidPull,
   type PlanningActionParamAssert,
   type PlanningActionParamError,
   type PlanningActionParamHover,
@@ -705,6 +707,57 @@ export class PageTaskExecutor {
             },
           };
         tasks.push(taskActionAndroidRecentAppsButton);
+      } else if (plan.type === 'AndroidLongPress') {
+        const taskActionAndroidLongPress: ExecutionTaskActionApply<PlanningActionParamAndroidLongPress> =
+          {
+            type: 'Action',
+            subType: 'AndroidLongPress',
+            param: plan.param as PlanningActionParamAndroidLongPress,
+            thought: plan.thought,
+            locate: plan.locate,
+            executor: async (param) => {
+              assert(
+                isAndroidPage(this.page),
+                'Cannot use long press on non-Android devices',
+              );
+              const { x, y, duration } = param;
+              await this.page.longPress(x, y, duration);
+            },
+          };
+        tasks.push(taskActionAndroidLongPress);
+      } else if (plan.type === 'AndroidPull') {
+        const taskActionAndroidPull: ExecutionTaskActionApply<PlanningActionParamAndroidPull> =
+          {
+            type: 'Action',
+            subType: 'AndroidPull',
+            param: plan.param as PlanningActionParamAndroidPull,
+            thought: plan.thought,
+            locate: plan.locate,
+            executor: async (param) => {
+              assert(
+                isAndroidPage(this.page),
+                'Cannot use pull action on non-Android devices',
+              );
+              const { direction, startPoint, distance, duration } = param;
+
+              const convertedStartPoint = startPoint
+                ? { left: startPoint.x, top: startPoint.y }
+                : undefined;
+
+              if (direction === 'down') {
+                await this.page.pullDown(
+                  convertedStartPoint,
+                  distance,
+                  duration,
+                );
+              } else if (direction === 'up') {
+                await this.page.pullUp(convertedStartPoint, distance, duration);
+              } else {
+                throw new Error(`Unknown pull direction: ${direction}`);
+              }
+            },
+          };
+        tasks.push(taskActionAndroidPull);
       } else {
         throw new Error(`Unknown or unsupported task type: ${plan.type}`);
       }