feat(ios, android): add screenshot resize scale option for Android and iOS agents

Author: quanru

.husky/commit-msg

@@ -1 +1,6 @@
-npx --no -- commitlint --edit "$1" 
+#!/bin/sh
+
+# Ensure node is in PATH (for fnm users)
+export PATH="$HOME/Library/Application Support/fnm/aliases/default/bin:$PATH"
+
+npx --no -- commitlint --edit "$1"

apps/site/docs/en/integrate-with-android.mdx

@@ -128,6 +128,7 @@ The AndroidDevice constructor supports the following parameters:
   - `remoteAdbPort?: number` - Optional, the remote adb port.
   - `imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii'` - Optional, when should Midscene invoke [yadb](https://github.com/ysbing/YADB) to input texts. (Default: 'always-yadb')
   - `displayId?: number` - Optional, the display id to use. (Default: undefined, means use the current display)
+  - `screenshotResizeScale?: number` - Optional, screenshot resize scale. For example, 0.5 means resize the screenshot to 50% of its original size, which can reduce AI processing image size and improve response speed. Default is `1 / devicePixelRatio` (automatically scaled based on device pixel ratio).
 
 ### Additional Android Agent Interfaces
 

apps/site/docs/en/integrate-with-ios.mdx

@@ -151,6 +151,7 @@ The IOSDevice constructor supports the following parameters:
   - `wdaPort?: number` - Optional, WebDriverAgent port. Default is 8100.
   - `wdaHost?: string` - Optional, WebDriverAgent host. Default is 'localhost'.
   - `autoDismissKeyboard?: boolean` - Optional, whether to automatically dismiss keyboard after text input. Default is true.
+  - `screenshotResizeScale?: number` - Optional, screenshot resize scale. For example, 0.5 means resize the screenshot to 50% of its original size, which can reduce AI processing image size and improve response speed. Default is 1 (no scaling).
   - `customActions?: DeviceAction<any>[]` - Optional, list of custom device actions.
 
 ### Additional iOS Agent Interfaces

apps/site/docs/zh/integrate-with-android.mdx

@@ -127,6 +127,7 @@ AndroidDevice 的构造函数支持以下参数：
   - `remoteAdbPort?: number` - 可选参数，用于指定远程 adb 端口。
   - `imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii'` - 可选参数，控制 Midscene 何时调用 [yadb](https://github.com/ysbing/YADB) 来输入文本。默认值为 'always-yadb'。
   - `displayId?: number` - 可选参数，用于指定要使用的显示器 ID。默认值为 undefined，表示使用当前显示器。
+  - `screenshotResizeScale?: number` - 可选参数，截图缩放比例。例如 0.5 表示将截图缩小到原来的 50%，可以减少 AI 处理的图片大小，提升响应速度。默认值为 `1 / devicePixelRatio`（根据设备像素比自动缩放）。
 
 ### Android Agent 上的更多接口
 

apps/site/docs/zh/integrate-with-ios.mdx

@@ -199,6 +199,7 @@ IOSDevice 的构造函数支持以下参数：
   - `wdaPort?: number` - 可选参数，WebDriverAgent 端口。默认值为 8100。
   - `wdaHost?: string` - 可选参数，WebDriverAgent 主机。默认值为 'localhost'。
   - `autoDismissKeyboard?: boolean` - 可选参数，是否在输入文本后自动关闭键盘。默认值为 true。
+  - `screenshotResizeScale?: number` - 可选参数，截图缩放比例。例如 0.5 表示将截图缩小到原来的 50%，可以减少 AI 处理的图片大小，提升响应速度。默认值为 1（不缩放）。
   - `customActions?: DeviceAction<any>[]` - 可选参数，自定义设备动作列表。
 
 ### iOS Agent 上的更多接口

packages/android/src/agent.ts

@@ -44,6 +44,7 @@ export async function agentFromAdbDevice(
     usePhysicalDisplayIdForScreenshot: opts?.usePhysicalDisplayIdForScreenshot,
     usePhysicalDisplayIdForDisplayLookup:
       opts?.usePhysicalDisplayIdForDisplayLookup,
+    screenshotResizeScale: opts?.screenshotResizeScale,
   });
 
   await device.connect();

packages/ios/src/agent.ts

@@ -31,6 +31,7 @@ export async function agentFromWebDriverAgent(
     wdaPort: opts?.wdaPort,
     wdaHost: opts?.wdaHost,
     useWDA: opts?.useWDA,
+    screenshotResizeScale: opts?.screenshotResizeScale,
   });
 
   await device.connect();

packages/ios/src/device.ts

@@ -37,12 +37,14 @@ export type IOSDeviceOpt = {
   wdaPort?: number;
   wdaHost?: string;
   useWDA?: boolean;
+  screenshotResizeScale?: number;
 } & IOSDeviceInputOpt;
 
 export class IOSDevice implements AbstractInterface {
   private deviceId: string;
   private devicePixelRatio = 1;
   private devicePixelRatioInitialized = false;
+  private scalingRatio = 1; // Record scaling ratio for coordinate adjustment
   private destroyed = false;
   private description: string | undefined;
   private customActions?: DeviceAction<any>[];
@@ -341,12 +343,35 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
   }
 
   async size(): Promise<Size> {
+    // Ensure device pixel ratio is initialized first
+    await this.initializeDevicePixelRatio();
+
     const screenSize = await this.getScreenSize();
 
+    // Determine scaling: use screenshotResizeScale if provided, otherwise use 1/devicePixelRatio
+    // Default is 1
+    const scale = this.options?.screenshotResizeScale ?? 1;
+    this.scalingRatio = scale;
+
+    // Apply scale to get logical dimensions for AI processing
+    const logicalWidth = Math.round(screenSize.width * scale);
+    const logicalHeight = Math.round(screenSize.height * scale);
+
+    debugDevice(
+      `size() - screenSize: ${screenSize.width}x${screenSize.height}, scale: ${scale}, logicalSize: ${logicalWidth}x${logicalHeight}`,
+    );
+
     return {
-      width: screenSize.width,
-      height: screenSize.height,
-      dpr: screenSize.scale,
+      width: logicalWidth,
+      height: logicalHeight,
+    };
+  }
+
+  private adjustCoordinates(x: number, y: number): { x: number; y: number } {
+    const scale = this.scalingRatio;
+    return {
+      x: Math.round(x / scale),
+      y: Math.round(y / scale),
     };
   }
 
@@ -399,7 +424,11 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
 
   // Core interaction methods
   async tap(x: number, y: number): Promise<void> {
-    await this.wdaBackend.tap(x, y);
+    const adjusted = this.adjustCoordinates(x, y);
+    debugDevice(
+      `tap at coordinates - input: (${x}, ${y}), adjusted: (${adjusted.x}, ${adjusted.y}), scale: ${this.scalingRatio}`,
+    );
+    await this.wdaBackend.tap(adjusted.x, adjusted.y);
   }
 
   // Android-compatible method name
@@ -409,11 +438,13 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
   }
 
   async doubleTap(x: number, y: number): Promise<void> {
-    await this.wdaBackend.doubleTap(x, y);
+    const adjusted = this.adjustCoordinates(x, y);
+    await this.wdaBackend.doubleTap(adjusted.x, adjusted.y);
   }
 
   async longPress(x: number, y: number, duration = 1000): Promise<void> {
-    await this.wdaBackend.longPress(x, y, duration);
+    const adjusted = this.adjustCoordinates(x, y);
+    await this.wdaBackend.longPress(adjusted.x, adjusted.y, duration);
   }
 
   async swipe(
@@ -423,7 +454,15 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
     toY: number,
     duration = 500,
   ): Promise<void> {
-    await this.wdaBackend.swipe(fromX, fromY, toX, toY, duration);
+    const adjustedFrom = this.adjustCoordinates(fromX, fromY);
+    const adjustedTo = this.adjustCoordinates(toX, toY);
+    await this.wdaBackend.swipe(
+      adjustedFrom.x,
+      adjustedFrom.y,
+      adjustedTo.x,
+      adjustedTo.y,
+      duration,
+    );
   }
 
   async typeText(text: string, options?: IOSDeviceInputOpt): Promise<void> {