feat(core): add screenshot scaling support for improved AI processing

Author: quanru

packages/android/src/device.ts

@@ -713,28 +713,15 @@ ${Object.keys(size)
     const width = Number.parseInt(match[isLandscape ? 2 : 1], 10);
     const height = Number.parseInt(match[isLandscape ? 1 : 2], 10);
 
-    // Use cached device pixel ratio instead of calling getDisplayDensity() every time
-
-    // Convert physical pixels to logical pixels for consistent coordinate system
-    // adjustCoordinates() will convert back to physical pixels when needed for touch operations
-    const logicalWidth = Math.round(width / this.devicePixelRatio);
-    const logicalHeight = Math.round(height / this.devicePixelRatio);
-
+    // Return physical pixels to match screenshot dimensions
+    // This ensures AI coordinate conversion uses the same dimensions as the screenshot
     return {
-      width: logicalWidth,
-      height: logicalHeight,
+      width,
+      height,
       dpr: this.devicePixelRatio,
     };
   }
 
-  private adjustCoordinates(x: number, y: number): { x: number; y: number } {
-    const ratio = this.devicePixelRatio;
-    return {
-      x: Math.round(x * ratio),
-      y: Math.round(y * ratio),
-    };
-  }
-
   /**
    * Calculate the end point for scroll operations based on start point, scroll delta, and screen boundaries.
    * This method ensures that scroll operations stay within screen bounds and maintain a minimum scroll distance
@@ -1183,20 +1170,17 @@ ${Object.keys(size)
   async mouseClick(x: number, y: number): Promise<void> {
     const adb = await this.getAdb();
 
-    // Use adjusted coordinates
-    const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
     await adb.shell(
-      `input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} 150`,
+      `input${this.getDisplayArg()} swipe ${x} ${y} ${x} ${y} 150`,
     );
   }
 
   async mouseDoubleClick(x: number, y: number): Promise<void> {
     const adb = await this.getAdb();
-    const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
 
     // Use input tap for double-click as it generates proper touch events
     // that Android can recognize as a double-click gesture
-    const tapCommand = `input${this.getDisplayArg()} tap ${adjustedX} ${adjustedY}`;
+    const tapCommand = `input${this.getDisplayArg()} tap ${x} ${y}`;
     await adb.shell(tapCommand);
     // Short delay between taps for double-click recognition
     await sleep(50);
@@ -1216,15 +1200,11 @@ ${Object.keys(size)
   ): Promise<void> {
     const adb = await this.getAdb();
 
-    // Use adjusted coordinates
-    const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
-    const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
-
     // Ensure duration has a default value
     const swipeDuration = duration ?? defaultNormalScrollDuration;
 
     await adb.shell(
-      `input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`,
+      `input${this.getDisplayArg()} swipe ${from.x} ${from.y} ${to.x} ${to.y} ${swipeDuration}`,
     );
   }
 
@@ -1264,22 +1244,12 @@ ${Object.keys(size)
     const endX = startX - deltaX;
     const endY = startY - deltaY;
 
-    // Adjust coordinates to fit device ratio
-    const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
-      startX,
-      startY,
-    );
-    const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(
-      endX,
-      endY,
-    );
-
     const adb = await this.getAdb();
     const swipeDuration = duration ?? defaultNormalScrollDuration;
 
     // Execute the swipe operation
     await adb.shell(
-      `input${this.getDisplayArg()} swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`,
+      `input${this.getDisplayArg()} swipe ${startX} ${startY} ${endX} ${endY} ${swipeDuration}`,
     );
   }
 
@@ -1320,10 +1290,8 @@ ${Object.keys(size)
   async longPress(x: number, y: number, duration = 1000): Promise<void> {
     const adb = await this.getAdb();
 
-    // Use adjusted coordinates
-    const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
     await adb.shell(
-      `input${this.getDisplayArg()} swipe ${adjustedX} ${adjustedY} ${adjustedX} ${adjustedY} ${duration}`,
+      `input${this.getDisplayArg()} swipe ${x} ${y} ${x} ${y} ${duration}`,
     );
   }
 
@@ -1355,13 +1323,9 @@ ${Object.keys(size)
   ): Promise<void> {
     const adb = await this.getAdb();
 
-    // Use adjusted coordinates
-    const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
-    const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
-
     // Use the specified duration for better pull gesture recognition
     await adb.shell(
-      `input${this.getDisplayArg()} swipe ${fromX} ${fromY} ${toX} ${toY} ${duration}`,
+      `input${this.getDisplayArg()} swipe ${from.x} ${from.y} ${to.x} ${to.y} ${duration}`,
     );
   }
 

packages/core/src/agent/agent.ts

@@ -48,6 +48,7 @@ import {
   globalConfigManager,
   globalModelConfigManager,
 } from '@midscene/shared/env';
+import { resizeImgBase64 } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 // import type { AndroidDeviceInputOpt } from '../device';
@@ -134,6 +135,11 @@ export class Agent<
    */
   private hasWarnedNonVLModel = false;
 
+  /**
+   * Screenshot scale factor for AI model processing
+   */
+  private screenshotScale?: number;
+
   // @deprecated use .interface instead
   get page() {
     return this.interface;
@@ -176,6 +182,7 @@ export class Agent<
       ? new ModelConfigManager(opts.modelConfig)
       : globalModelConfigManager;
 
+    this.screenshotScale = opts?.screenshotScale;
     this.onTaskStartTip = this.opts.onTaskStartTip;
 
     this.insight = new Insight(async (action: InsightAction) => {
@@ -218,15 +225,48 @@ export class Agent<
       return this.frozenUIContext;
     }
 
+    // Get original context
+    let context: UIContext;
     if (this.interface.getContext) {
       debug('Using page.getContext for action:', action);
-      return await this.interface.getContext();
+      context = await this.interface.getContext();
     } else {
       debug('Using commonContextParser for action:', action);
-      return await commonContextParser(this.interface, {
+      context = await commonContextParser(this.interface, {
         uploadServerUrl: this.modelConfigManager.getUploadTestServerUrl(),
       });
     }
+
+    // Unified screenshot scaling: prioritize screenshotScale, otherwise use DPR
+    let targetWidth = context.size.width;
+    let targetHeight = context.size.height;
+    let needResize = false;
+
+    if (this.screenshotScale && this.screenshotScale !== 1) {
+      // User-specified scaling ratio
+      debug(`Applying user screenshot scale: ${this.screenshotScale}`);
+      targetWidth = Math.round(context.size.width * this.screenshotScale);
+      targetHeight = Math.round(context.size.height * this.screenshotScale);
+      needResize = true;
+    } else if (context.size.dpr && context.size.dpr !== 1) {
+      // No user-specified scaling, use DPR scaling to logical size
+      debug(
+        `Applying DPR scaling: ${context.size.dpr} (resize to logical size)`,
+      );
+      // Target is logical size, no need to change targetWidth/targetHeight
+      needResize = true;
+    }
+
+    // Execute scaling
+    if (needResize) {
+      debug(`Resizing screenshot to ${targetWidth}x${targetHeight}`);
+      context.screenshotBase64 = await resizeImgBase64(
+        context.screenshotBase64,
+        { width: targetWidth, height: targetHeight },
+      );
+    }
+
+    return context;
   }
 
   async _snapshotContext(): Promise<UIContext> {
@@ -830,9 +870,9 @@ export class Agent<
     return {
       rect: element?.rect,
       center: element?.center,
-      scale: (await this.interface.size()).dpr,
+      dpr: (await this.interface.size()).dpr,
     } as Pick<LocateResultElement, 'rect' | 'center'> & {
-      scale: number;
+      dpr: number;
     };
   }
 

packages/core/src/agent/utils.ts

@@ -50,21 +50,12 @@ export async function commonContextParser(
   });
   debugProfile('UploadTestInfoToServer end');
 
-  let screenshotBase64 = await interfaceInstance.screenshotBase64();
+  const screenshotBase64 = await interfaceInstance.screenshotBase64();
   assert(screenshotBase64!, 'screenshotBase64 is required');
 
   const size = await interfaceInstance.size();
   debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);
 
-  if (size.dpr && size.dpr !== 1) {
-    debugProfile('Resizing screenshot for non-1 dpr');
-    screenshotBase64 = await resizeImgBase64(screenshotBase64, {
-      width: size.width,
-      height: size.height,
-    });
-    debugProfile('ResizeImgBase64 end');
-  }
-
   return {
     tree: {
       node: null,

packages/core/src/types.ts

@@ -604,4 +604,6 @@ export interface AgentOpt {
   modelConfig?: TModelConfigFn;
   cache?: Cache;
   replanningCycleLimit?: number;
+  /* Screenshot scaling ratio to reduce image size sent to AI for better performance */
+  screenshotScale?: number;
 }

packages/shared/src/img/info.ts

@@ -1,14 +1,9 @@
 import assert from 'node:assert';
 import { Buffer } from 'node:buffer';
 import type Jimp from 'jimp';
+import type { Size } from '../types';
 import getJimp from './get-jimp';
 
-export interface Size {
-  width: number;
-  height: number;
-  dpr?: number;
-}
-
 export interface ImageInfo extends Size {
   jimpImage: Jimp;
 }

packages/shared/src/types/index.ts

@@ -7,9 +7,9 @@ export interface Point {
 }
 
 export interface Size {
-  width: number; // device independent window size
+  width: number; // logical pixel size
   height: number;
-  dpr?: number; // the scale factor of the screenshots
+  dpr?: number; // dpr is the ratio of the physical pixel to the logical pixel. For example, the dpr is 2 when the screenshotBase64 returned is 2000x1000 when the logical width and height are 1000x500 here. Overriding the dpr will affect how the screenshotBase64 is resized before being sent to the AI model.
 }
 
 export type Rect = Point & Size & { zoom?: number };