fix(android): simplify scroll methods by removing startPoint parameter and enhancing scrolling logic (#1039)

* fix(android): simplify scroll methods by removing startPoint parameter and enhancing scrolling logic

* fix(android): enhance scrolling methods to support startPoint parameter and improve swipe duration handling

* docs(site): update acknowledgments to include Rslib as a build tool

Author: quanru

apps/site/docs/en/index.mdx

@@ -108,7 +108,7 @@ There are so many UI automation tools out there, and each one seems to be all-po
 
 We would like to thank the following projects:
 
-- [Rsbuild](https://github.com/web-infra-dev/rsbuild) for the build tool.
+- [Rsbuild](https://github.com/web-infra-dev/rsbuild) and [Rslib](https://github.com/web-infra-dev/rslib) for the build tool.
 - [UI-TARS](https://github.com/bytedance/ui-tars) for the open-source agent model UI-TARS.
 - [Qwen2.5-VL](https://github.com/QwenLM/Qwen2.5-VL) for the open-source VL model Qwen2.5-VL.
 - [scrcpy](https://github.com/Genymobile/scrcpy) and [yume-chan](https://github.com/yume-chan) allow us to control Android devices with browser.

apps/site/docs/zh/index.mdx

@@ -105,7 +105,7 @@ for (const record of recordList) {
 
 我们要感谢以下项目：
 
-- [Rsbuild](https://github.com/web-infra-dev/rsbuild) 提供构建工具。
+- [Rsbuild](https://github.com/web-infra-dev/rsbuild) 和 [Rslib](https://github.com/web-infra-dev/rslib) 提供构建工具。
 - [UI-TARS](https://github.com/bytedance/ui-tars) 提供开源智能体模型 UI-TARS。
 - [Qwen2.5-VL](https://github.com/QwenLM/Qwen2.5-VL) 提供开源 VL 模型 Qwen2.5-VL。
 - [scrcpy](https://github.com/Genymobile/scrcpy) 和 [yume-chan](https://github.com/yume-chan) 让我们能够用浏览器控制 Android 设备。

packages/android/src/page/index.ts

@@ -522,11 +522,14 @@ ${Object.keys(size)
   get mouse() {
     return {
       click: (x: number, y: number) => this.mouseClick(x, y),
-      wheel: (deltaX: number, deltaY: number) =>
-        this.mouseWheel(deltaX, deltaY),
+      wheel: (deltaX: number, deltaY: number, duration?: number) =>
+        this.mouseWheel(deltaX, deltaY, duration),
       move: (x: number, y: number) => this.mouseMove(x, y),
-      drag: (from: { x: number; y: number }, to: { x: number; y: number }) =>
-        this.mouseDrag(from, to),
+      drag: (
+        from: { x: number; y: number },
+        to: { x: number; y: number },
+        duration?: number,
+      ) => this.mouseDrag(from, to, duration),
     };
   }
 
@@ -582,59 +585,74 @@ ${Object.keys(size)
 
   async scrollUntilTop(startPoint?: Point): Promise<void> {
     if (startPoint) {
+      const { height } = await this.size();
       const start = { x: startPoint.left, y: startPoint.top };
-      const end = { x: start.x, y: 0 };
+      const end = { x: start.x, y: height };
 
-      await this.mouseDrag(start, end);
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
       return;
     }
 
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(0, 9999999, defaultFastScrollDuration),
+      this.mouseWheel(0, -9999999, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
   async scrollUntilBottom(startPoint?: Point): Promise<void> {
     if (startPoint) {
-      const { height } = await this.size();
       const start = { x: startPoint.left, y: startPoint.top };
-      const end = { x: start.x, y: height };
-      await this.mouseDrag(start, end);
+      const end = { x: start.x, y: 0 };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
       return;
     }
 
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(0, -9999999, defaultFastScrollDuration),
+      this.mouseWheel(0, 9999999, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
   async scrollUntilLeft(startPoint?: Point): Promise<void> {
     if (startPoint) {
+      const { width } = await this.size();
       const start = { x: startPoint.left, y: startPoint.top };
-      const end = { x: 0, y: start.y };
-      await this.mouseDrag(start, end);
+      const end = { x: width, y: start.y };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
       return;
     }
 
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(9999999, 0, defaultFastScrollDuration),
+      this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
 
   async scrollUntilRight(startPoint?: Point): Promise<void> {
     if (startPoint) {
-      const { width } = await this.size();
       const start = { x: startPoint.left, y: startPoint.top };
-      const end = { x: width, y: start.y };
-      await this.mouseDrag(start, end);
+      const end = { x: 0, y: start.y };
+
+      await repeat(defaultScrollUntilTimes, () =>
+        this.mouseDrag(start, end, defaultFastScrollDuration),
+      );
+      await sleep(1000);
       return;
     }
 
     await repeat(defaultScrollUntilTimes, () =>
-      this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
+      this.mouseWheel(9999999, 0, defaultFastScrollDuration),
     );
     await sleep(1000);
   }
@@ -645,13 +663,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endY = Math.max(0, start.y - scrollDistance);
+      const endY = Math.min(height, start.y + scrollDistance);
       const end = { x: start.x, y: endY };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(0, scrollDistance);
+    await this.mouseWheel(0, -scrollDistance);
   }
 
   async scrollDown(distance?: number, startPoint?: Point): Promise<void> {
@@ -660,13 +678,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endY = Math.min(height, start.y + scrollDistance);
+      const endY = Math.max(0, start.y - scrollDistance);
       const end = { x: start.x, y: endY };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(0, -scrollDistance);
+    await this.mouseWheel(0, scrollDistance);
   }
 
   async scrollLeft(distance?: number, startPoint?: Point): Promise<void> {
@@ -675,13 +693,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endX = Math.max(0, start.x - scrollDistance);
+      const endX = Math.min(width, start.x + scrollDistance);
       const end = { x: endX, y: start.y };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(scrollDistance, 0);
+    await this.mouseWheel(-scrollDistance, 0);
   }
 
   async scrollRight(distance?: number, startPoint?: Point): Promise<void> {
@@ -690,13 +708,13 @@ ${Object.keys(size)
 
     if (startPoint) {
       const start = { x: startPoint.left, y: startPoint.top };
-      const endX = Math.min(width, start.x + scrollDistance);
+      const endX = Math.max(0, start.x - scrollDistance);
       const end = { x: endX, y: start.y };
       await this.mouseDrag(start, end);
       return;
     }
 
-    await this.mouseWheel(-scrollDistance, 0);
+    await this.mouseWheel(scrollDistance, 0);
   }
 
   private async ensureYadb() {
@@ -807,20 +825,26 @@ ${Object.keys(size)
   private async mouseDrag(
     from: { x: number; y: number },
     to: { x: number; y: number },
+    duration?: number,
   ): Promise<void> {
     const adb = await this.getAdb();
 
     // Use adjusted coordinates
     const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
     const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
 
-    await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
+    // Ensure duration has a default value
+    const swipeDuration = duration ?? 300;
+
+    await adb.shell(
+      `input swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`,
+    );
   }
 
   private async mouseWheel(
     deltaX: number,
     deltaY: number,
-    duration = defaultNormalScrollDuration,
+    duration?: number,
   ): Promise<void> {
     const { width, height } = await this.size();
 
@@ -842,8 +866,11 @@ ${Object.keys(size)
     deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
 
     // Calculate the end coordinates
-    const endX = startX + deltaX;
-    const endY = startY + deltaY;
+    // Note: For swipe, we need to reverse the delta direction
+    // because positive deltaY should scroll up (show top content),
+    // which requires swiping from bottom to top (decreasing Y)
+    const endX = startX - deltaX;
+    const endY = startY - deltaY;
 
     // Adjust coordinates to fit device ratio
     const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
@@ -857,9 +884,12 @@ ${Object.keys(size)
 
     const adb = await this.getAdb();
 
+    // Ensure duration has a default value
+    const swipeDuration = duration ?? defaultNormalScrollDuration;
+
     // Execute the swipe operation
     await adb.shell(
-      `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`,
+      `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`,
     );
   }
 

packages/android/tests/ai/setting.test.ts

@@ -16,13 +16,16 @@ describe(
       });
 
       await agent.launch('com.android.settings/.Settings');
-
       await agent.aiAction('scroll list to bottom');
       await agent.aiAction('open "More settings"');
-      await agent.aiAction('scroll list to bottom');
+      await agent.aiAction('scroll left until left edge');
+      await agent.aiAction('scroll right until right edge');
       await agent.aiAction('scroll list to top');
-      await agent.aiAction('swipe down one screen');
-      await agent.aiAction('swipe up one screen');
+      await agent.aiAction('scroll list to bottom');
+      await agent.aiAction('scroll down one screen');
+      await agent.aiAction('scroll up one screen');
+      await agent.aiAction('scroll right one screen');
+      await agent.aiAction('scroll left one screen');
     });
   },
   360 * 1000,