fix(android): enhance scrolling methods to support startPoint parameter and improve swipe duration handling

Author: quanru

packages/android/src/page/index.ts

@@ -471,11 +471,14 @@ ${Object.keys(size)
   get mouse() {
     return {
       click: (x: number, y: number) => this.mouseClick(x, y),
-      wheel: (deltaX: number, deltaY: number) =>
-        this.mouseWheel(deltaX, deltaY),
+      wheel: (deltaX: number, deltaY: number, duration?: number) =>
+        this.mouseWheel(deltaX, deltaY, duration),
       move: (x: number, y: number) => this.mouseMove(x, y),
-      drag: (from: { x: number; y: number }, to: { x: number; y: number }) =>
-        this.mouseDrag(from, to),
+      drag: (
+        from: { x: number; y: number },
+        to: { x: number; y: number },
+        duration?: number,
+      ) => this.mouseDrag(from, to, duration),
     };
   }
 
@@ -529,30 +532,76 @@ ${Object.keys(size)
     return '';
   }
 
-  async scrollUntilTop(): Promise<void> {
+  async scrollUntilTop(startPoint?: Point): Promise<void> {
+    if (startPoint) {
+      const { height } = await this.size();
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: start.x, y: height };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
+      return;
+    }
+
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(0, 9999999, defaultFastScrollDuration),
+      this.mouseWheel(0, -9999999, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
-  async scrollUntilBottom(): Promise<void> {
+  async scrollUntilBottom(startPoint?: Point): Promise<void> {
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: start.x, y: 0 };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
+      return;
+    }
+
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(0, -9999999, defaultFastScrollDuration),
+      this.mouseWheel(0, 9999999, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
-  async scrollUntilLeft(): Promise<void> {
+  async scrollUntilLeft(startPoint?: Point): Promise<void> {
+    if (startPoint) {
+      const { width } = await this.size();
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: width, y: start.y };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
+      return;
+    }
+
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(9999999, 0, defaultFastScrollDuration),
+      this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
-  async scrollUntilRight(): Promise<void> {
+  async scrollUntilRight(startPoint?: Point): Promise<void> {
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: 0, y: start.y };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
+      return;
+    }
+
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
+      this.mouseWheel(9999999, 0, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
@@ -563,13 +612,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endY = Math.max(0, start.y - scrollDistance);
+      const endY = Math.min(height, start.y + scrollDistance);
       const end = { x: start.x, y: endY };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(0, scrollDistance);
+    await this.mouseWheel(0, -scrollDistance);
   }
 
   async scrollDown(distance?: number, startPoint?: Point): Promise<void> {
@@ -578,13 +627,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endY = Math.min(height, start.y + scrollDistance);
+      const endY = Math.max(0, start.y - scrollDistance);
       const end = { x: start.x, y: endY };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(0, -scrollDistance);
+    await this.mouseWheel(0, scrollDistance);
   }
 
   async scrollLeft(distance?: number, startPoint?: Point): Promise<void> {
@@ -593,13 +642,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endX = Math.max(0, start.x - scrollDistance);
+      const endX = Math.min(width, start.x + scrollDistance);
       const end = { x: endX, y: start.y };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(scrollDistance, 0);
+    await this.mouseWheel(-scrollDistance, 0);
   }
 
   async scrollRight(distance?: number, startPoint?: Point): Promise<void> {
@@ -608,13 +657,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endX = Math.min(width, start.x + scrollDistance);
+      const endX = Math.max(0, start.x - scrollDistance);
       const end = { x: endX, y: start.y };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(-scrollDistance, 0);
+    await this.mouseWheel(scrollDistance, 0);
   }
 
   private async ensureYadb() {
@@ -723,20 +772,26 @@ ${Object.keys(size)
   private async mouseDrag(
     from: { x: number; y: number },
     to: { x: number; y: number },
+    duration?: number,
   ): Promise<void> {
     const adb = await this.getAdb();
 
     // Use adjusted coordinates
     const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
     const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
 
-    await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
+    // Ensure duration has a default value
+    const swipeDuration = duration ?? 300;
+
+    await adb.shell(
+      `input swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`,
+    );
   }
 
   private async mouseWheel(
     deltaX: number,
     deltaY: number,
-    duration = defaultNormalScrollDuration,
+    duration?: number,
   ): Promise<void> {
     const { width, height } = await this.size();
 
@@ -758,8 +813,11 @@ ${Object.keys(size)
     deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
 
     // Calculate the end coordinates
-    const endX = startX + deltaX;
-    const endY = startY + deltaY;
+    // Note: For swipe, we need to reverse the delta direction
+    // because positive deltaY should scroll up (show top content),
+    // which requires swiping from bottom to top (decreasing Y)
+    const endX = startX - deltaX;
+    const endY = startY - deltaY;
 
     // Adjust coordinates to fit device ratio
     const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
@@ -773,9 +831,12 @@ ${Object.keys(size)
 
     const adb = await this.getAdb();
 
+    // Ensure duration has a default value
+    const swipeDuration = duration ?? defaultNormalScrollDuration;
+
     // Execute the swipe operation
     await adb.shell(
-      `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`,
+      `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`,
     );
   }
 

packages/android/tests/ai/setting.test.ts

@@ -16,13 +16,16 @@ describe(
       });
 
       await agent.launch('com.android.settings/.Settings');
-
       await agent.aiAction('scroll list to bottom');
       await agent.aiAction('open "More settings"');
-      await agent.aiAction('scroll list to bottom');
+      await agent.aiAction('scroll left until left edge');
+      await agent.aiAction('scroll right until right edge');
       await agent.aiAction('scroll list to top');
-      await agent.aiAction('swipe down one screen');
-      await agent.aiAction('swipe up one screen');
+      await agent.aiAction('scroll list to bottom');
+      await agent.aiAction('scroll down one screen');
+      await agent.aiAction('scroll up one screen');
+      await agent.aiAction('scroll right one screen');
+      await agent.aiAction('scroll left one screen');
     });
   },
   360 * 1000,