feat(android): remove screenshotResizeRatio option and related logic from AndroidDevice and tests

Author: quanru

packages/android/src/device.ts

@@ -60,7 +60,6 @@ export type AndroidDeviceOpt = {
   usePhysicalDisplayIdForScreenshot?: boolean;
   usePhysicalDisplayIdForDisplayLookup?: boolean;
   customActions?: DeviceAction<any>[];
-  screenshotResizeRatio?: number;
 } & AndroidDeviceInputOpt;
 
 export class AndroidDevice implements AbstractInterface {

packages/android/tests/ai/setting.test.ts

@@ -1,3 +1,4 @@
+import { sleep } from '@midscene/core/utils';
 import { describe, it, vi } from 'vitest';
 import { agentFromAdbDevice, getConnectedDevices } from '../../src';
 
@@ -11,25 +12,14 @@ describe(
     await it('Android settings page demo for scroll', async () => {
       const devices = await getConnectedDevices();
       const agent = await agentFromAdbDevice(devices[0].udid, {
+        // scale: 0.5,
         aiActionContext:
           'If any location, permission, user agreement, etc. popup, click agree. If login page pops up, close it.',
       });
 
       await agent.launch('com.android.settings/.Settings');
-      await agent.aiAction('pull down to refresh');
-      await agent.aiAction('long press chat list first chat');
-      await agent.aiAction('click recent apps button');
-      await agent.aiAction('click android home button');
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('open "More settings"');
-      await agent.aiAction('scroll left until left edge');
-      await agent.aiAction('scroll right until right edge');
-      await agent.aiAction('scroll list to top');
-      await agent.aiAction('scroll list to bottom');
-      await agent.aiAction('scroll down one screen');
-      await agent.aiAction('scroll up one screen');
-      await agent.aiAction('scroll right one screen');
-      await agent.aiAction('scroll left one screen');
+      await sleep(2000);
+      await agent.aiAction('点击蓝牙');
     });
   },
   360 * 1000,

packages/android/tests/unit-test/page.test.ts

@@ -234,94 +234,6 @@ describe('AndroidDevice', () => {
       expect(result).toContain(mockBuffer.toString('base64'));
       expect(mockAdb.shell).toHaveBeenCalledWith(expect.stringMatching(/rm/));
     });
-
-    it('should apply custom resize ratio when screenshotResizeRatio is set', async () => {
-      const customDevice = new AndroidDevice('test-device', {
-        screenshotResizeRatio: 0.5,
-      });
-
-      vi.spyOn(customDevice, 'size').mockResolvedValue({
-        width: 1080,
-        height: 1920,
-        dpr: 2,
-      });
-
-      const mockBuffer = Buffer.from('test-screenshot');
-      mockAdb.takeScreenshot.mockResolvedValue(mockBuffer);
-      vi.spyOn(customDevice, 'getAdb').mockResolvedValue(mockAdb);
-
-      // Mock createImgBase64ByFormat
-      vi.spyOn(ImgUtils, 'createImgBase64ByFormat').mockReturnValue(
-        `data:image/png;base64,${mockBuffer.toString('base64')}`,
-      );
-
-      await customDevice.screenshotBase64();
-
-      // Verify that resizeAndConvertImgBuffer was called with half the original size
-      expect(ImgUtils.resizeAndConvertImgBuffer).toHaveBeenCalledWith(
-        'png',
-        mockBuffer,
-        {
-          width: 540, // 1080 * 0.5
-          height: 960, // 1920 * 0.5
-        },
-      );
-    });
-
-    it('should use original size when screenshotResizeRatio is not set', async () => {
-      const mockBuffer = Buffer.from('test-screenshot');
-      mockAdb.takeScreenshot.mockResolvedValue(mockBuffer);
-
-      // Mock createImgBase64ByFormat
-      vi.spyOn(ImgUtils, 'createImgBase64ByFormat').mockReturnValue(
-        `data:image/png;base64,${mockBuffer.toString('base64')}`,
-      );
-
-      await device.screenshotBase64();
-
-      // Verify that resizeAndConvertImgBuffer was called with original size
-      expect(ImgUtils.resizeAndConvertImgBuffer).toHaveBeenCalledWith(
-        'png',
-        mockBuffer,
-        {
-          width: 1080, // original width
-          height: 1920, // original height
-        },
-      );
-    });
-
-    it('should use default ratio of 1.0 when screenshotResizeRatio is undefined', async () => {
-      const customDevice = new AndroidDevice('test-device', {
-        screenshotResizeRatio: undefined,
-      });
-
-      vi.spyOn(customDevice, 'size').mockResolvedValue({
-        width: 1080,
-        height: 1920,
-        dpr: 2,
-      });
-
-      const mockBuffer = Buffer.from('test-screenshot');
-      mockAdb.takeScreenshot.mockResolvedValue(mockBuffer);
-      vi.spyOn(customDevice, 'getAdb').mockResolvedValue(mockAdb);
-
-      // Mock createImgBase64ByFormat
-      vi.spyOn(ImgUtils, 'createImgBase64ByFormat').mockReturnValue(
-        `data:image/png;base64,${mockBuffer.toString('base64')}`,
-      );
-
-      await customDevice.screenshotBase64();
-
-      // Verify that resizeAndConvertImgBuffer was called with original size (ratio 1.0)
-      expect(ImgUtils.resizeAndConvertImgBuffer).toHaveBeenCalledWith(
-        'png',
-        mockBuffer,
-        {
-          width: 1080, // 1080 * 1.0
-          height: 1920, // 1920 * 1.0
-        },
-      );
-    });
   });
 
   describe('mouse', () => {

packages/core/src/agent/agent.ts

@@ -50,7 +50,6 @@ import {
 } from '@midscene/shared/env';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
-// import type { AndroidDeviceInputOpt } from '../device';
 import { TaskCache } from './task-cache';
 import { TaskExecutor, locatePlanForLocate } from './tasks';
 import { locateParamStr, paramStr, taskTitleStr, typeStr } from './ui-utils';
@@ -59,6 +58,7 @@ import {
   getReportFileName,
   parsePrompt,
   printReportMsg,
+  scaleElementCoordinates,
 } from './utils';
 import { trimContextByViewport } from './utils';
 
@@ -113,6 +113,11 @@ export class Agent<
 
   modelConfigManager: ModelConfigManager;
 
+  /**
+   * Scale factor for screenshot processing
+   */
+  private scale: number | undefined;
+
   /**
    * Frozen page context for consistent AI operations
    */
@@ -165,6 +170,8 @@ export class Agent<
       ? new ModelConfigManager(opts.modelConfig)
       : globalModelConfigManager;
 
+    this.scale = opts?.scale;
+
     this.onTaskStartTip = this.opts.onTaskStartTip;
 
     this.insight = new Insight(async (action: InsightAction) => {
@@ -187,6 +194,7 @@ export class Agent<
       taskCache: this.taskCache,
       onTaskStart: this.callbackOnTaskStartTip.bind(this),
       replanningCycleLimit: this.opts.replanningCycleLimit,
+      scale: this.scale,
     });
     this.dump = this.resetDump();
     this.reportFileName =
@@ -215,6 +223,7 @@ export class Agent<
       debug('Using commonContextParser for action:', action);
       return await commonContextParser(this.interface, {
         uploadServerUrl: this.modelConfigManager.getUploadTestServerUrl(),
+        customScale: this.scale,
       });
     }
   }
@@ -817,10 +826,28 @@ export class Agent<
 
     const { element } = output;
 
+    if (!element) {
+      return {
+        rect: undefined,
+        center: undefined,
+        scale: (await this.interface.size()).dpr,
+      } as Pick<LocateResultElement, 'rect' | 'center'> & {
+        scale: number;
+      };
+    }
+
+    // Apply coordinate scaling using shared utility function
+    const deviceSize = await this.interface.size();
+    const scaledElement = scaleElementCoordinates(
+      element,
+      this.scale,
+      deviceSize.dpr,
+    );
+
     return {
-      rect: element?.rect,
-      center: element?.center,
-      scale: (await this.interface.size()).dpr,
+      rect: scaledElement.rect,
+      center: scaledElement.center,
+      scale: deviceSize.dpr,
     } as Pick<LocateResultElement, 'rect' | 'center'> & {
       scale: number;
     };

packages/core/src/agent/tasks.ts

@@ -54,6 +54,7 @@ import {
   matchElementFromCache,
   matchElementFromPlan,
   parsePrompt,
+  scaleElementCoordinates,
 } from './utils';
 
 interface ExecutionResult<OutputType = any> {
@@ -90,6 +91,8 @@ export class TaskExecutor {
 
   replanningCycleLimit?: number;
 
+  scale?: number;
+
   // @deprecated use .interface instead
   get page() {
     return this.interface;
@@ -102,11 +105,13 @@ export class TaskExecutor {
       taskCache?: TaskCache;
       onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
       replanningCycleLimit?: number;
+      scale?: number;
     },
   ) {
     this.interface = interfaceInstance;
     this.insight = insight;
     this.taskCache = opts.taskCache;
+    this.scale = opts.scale;
     this.onTaskStartCallback = opts?.onTaskStart;
     this.replanningCycleLimit = opts.replanningCycleLimit;
     this.conversationHistory = new ConversationHistory();
@@ -357,6 +362,15 @@ export class TaskExecutor {
             throw new Error(`Element not found: ${param.prompt}`);
           }
 
+          // Apply coordinate scaling using shared utility function
+          // At this point, element is guaranteed to be non-null due to the check above
+          const deviceSize = await this.interface.size();
+          const scaledElement = scaleElementCoordinates(
+            element!,
+            this.scale,
+            deviceSize.dpr,
+          );
+
           let hitBy: ExecutionTaskHitBy | undefined;
 
           if (userExpectedPathHitFlag) {
@@ -391,11 +405,11 @@ export class TaskExecutor {
             };
           }
 
-          onResult?.(element);
+          onResult?.(scaledElement);
 
           return {
             output: {
-              element,
+              element: scaledElement,
             },
             uiContext,
             hitBy,

packages/core/src/agent/utils.ts

@@ -6,6 +6,7 @@ import type {
   ExecutionDump,
   ExecutionTask,
   ExecutorContext,
+  LocateResultElement,
   PlanningLocateParam,
   TMultimodalPrompt,
   TUserPrompt,
@@ -32,7 +33,7 @@ const debugProfile = getDebug('web:tool:profile');
 
 export async function commonContextParser(
   interfaceInstance: AbstractInterface,
-  _opt: { uploadServerUrl?: string },
+  _opt: { uploadServerUrl?: string; customScale?: number },
 ): Promise<UIContext> {
   assert(interfaceInstance, 'interfaceInstance is required');
 
@@ -53,13 +54,27 @@ export async function commonContextParser(
   const size = await interfaceInstance.size();
   debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);
 
-  if (size.dpr && size.dpr !== 1) {
+  // Handle DPR scaling (original logic) and custom scaling
+  const customScale = _opt.customScale;
+
+  if (customScale && customScale !== 1) {
+    // Custom scale: scale the logical size
+    debugProfile(`Resizing screenshot with custom scale: ${customScale}`);
+    const targetWidth = Math.round(size.width * customScale);
+    const targetHeight = Math.round(size.height * customScale);
+    screenshotBase64 = await resizeImgBase64(screenshotBase64, {
+      width: targetWidth,
+      height: targetHeight,
+    });
+    debugProfile('Custom scale ResizeImgBase64 end');
+  } else if (size.dpr && size.dpr !== 1) {
+    // DPR scaling: resize physical screenshot to logical size
     debugProfile('Resizing screenshot for non-1 dpr');
     screenshotBase64 = await resizeImgBase64(screenshotBase64, {
       width: size.width,
       height: size.height,
     });
-    debugProfile('ResizeImgBase64 end');
+    debugProfile('DPR ResizeImgBase64 end');
   }
 
   return {
@@ -291,3 +306,39 @@ export const parsePrompt = (
       : undefined,
   };
 };
+
+/**
+ * Scale element coordinates from custom scaled screenshot space to device logical space
+ *
+ * This is only needed when a custom scale was explicitly provided. When no custom scale
+ * is set, devices handle DPR coordinate scaling naturally.
+ */
+export function scaleElementCoordinates(
+  element: LocateResultElement,
+  scale: number | undefined,
+  _deviceDpr: number | undefined,
+): LocateResultElement {
+  // Only apply coordinate scaling when a custom scale was explicitly provided
+  if (!scale || scale === 1) return element;
+
+  // Calculate coordinate scaling factor to convert back to logical space
+  const coordinateScale = 1 / scale;
+
+  return {
+    ...element,
+    rect: element.rect
+      ? {
+          left: Math.round(element.rect.left * coordinateScale),
+          top: Math.round(element.rect.top * coordinateScale),
+          width: Math.round(element.rect.width * coordinateScale),
+          height: Math.round(element.rect.height * coordinateScale),
+        }
+      : element.rect,
+    center: element.center
+      ? ([
+          Math.round(element.center[0] * coordinateScale),
+          Math.round(element.center[1] * coordinateScale),
+        ] as [number, number])
+      : element.center,
+  };
+}

packages/core/src/types.ts

@@ -594,4 +594,6 @@ export interface AgentOpt {
   modelConfig?: TModelConfigFn;
   useCache?: boolean;
   replanningCycleLimit?: number;
+  /* scale factor for screenshot processing, applied uniformly by Agent */
+  scale?: number;
 }