diff --git a/apps/site/docs/en/index.mdx b/apps/site/docs/en/index.mdx index b968d662f..cc79d6745 100644 --- a/apps/site/docs/en/index.mdx +++ b/apps/site/docs/en/index.mdx @@ -108,7 +108,7 @@ There are so many UI automation tools out there, and each one seems to be all-po We would like to thank the following projects: -- [Rsbuild](https://github.com/web-infra-dev/rsbuild) for the build tool. +- [Rsbuild](https://github.com/web-infra-dev/rsbuild) and [Rslib](https://github.com/web-infra-dev/rslib) for the build tool. - [UI-TARS](https://github.com/bytedance/ui-tars) for the open-source agent model UI-TARS. - [Qwen2.5-VL](https://github.com/QwenLM/Qwen2.5-VL) for the open-source VL model Qwen2.5-VL. - [scrcpy](https://github.com/Genymobile/scrcpy) and [yume-chan](https://github.com/yume-chan) allow us to control Android devices with browser. diff --git a/apps/site/docs/zh/index.mdx b/apps/site/docs/zh/index.mdx index af898fe86..e88ec84e4 100644 --- a/apps/site/docs/zh/index.mdx +++ b/apps/site/docs/zh/index.mdx @@ -105,7 +105,7 @@ for (const record of recordList) { 我们要感谢以下项目： -- [Rsbuild](https://github.com/web-infra-dev/rsbuild) 提供构建工具。 +- [Rsbuild](https://github.com/web-infra-dev/rsbuild) 和 [Rslib](https://github.com/web-infra-dev/rslib) 提供构建工具。 - [UI-TARS](https://github.com/bytedance/ui-tars) 提供开源智能体模型 UI-TARS。 - [Qwen2.5-VL](https://github.com/QwenLM/Qwen2.5-VL) 提供开源 VL 模型 Qwen2.5-VL。 - [scrcpy](https://github.com/Genymobile/scrcpy) 和 [yume-chan](https://github.com/yume-chan) 让我们能够用浏览器控制 Android 设备。 diff --git a/packages/android/src/page/index.ts b/packages/android/src/page/index.ts index 175dfa563..9e91d47fe 100644 --- a/packages/android/src/page/index.ts +++ b/packages/android/src/page/index.ts @@ -521,11 +521,14 @@ ${Object.keys(size) get mouse() { return { click: (x: number, y: number) => this.mouseClick(x, y), - wheel: (deltaX: number, deltaY: number) => - this.mouseWheel(deltaX, deltaY), + wheel: (deltaX: number, deltaY: number, duration?: number) => + this.mouseWheel(deltaX, deltaY, duration), move: (x: number, y: number) => this.mouseMove(x, y), - drag: (from: { x: number; y: number }, to: { x: number; y: number }) => - this.mouseDrag(from, to), + drag: ( + from: { x: number; y: number }, + to: { x: number; y: number }, + duration?: number, + ) => this.mouseDrag(from, to, duration), }; } @@ -581,59 +584,74 @@ ${Object.keys(size) async scrollUntilTop(startPoint?: Point): Promise { if (startPoint) { + const { height } = await this.size(); const start = { x: startPoint.left, y: startPoint.top }; - const end = { x: start.x, y: 0 }; + const end = { x: start.x, y: height }; - await this.mouseDrag(start, end); + await repeat(defaultScrollUntilTimes, () => + this.mouseDrag(start, end, defaultFastScrollDuration), + ); + await sleep(1000); return; } await repeat(defaultScrollUntilTimes, () => - this.mouseWheel(0, 9999999, defaultFastScrollDuration), + this.mouseWheel(0, -9999999, defaultFastScrollDuration), ); await sleep(1000); } async scrollUntilBottom(startPoint?: Point): Promise { if (startPoint) { - const { height } = await this.size(); const start = { x: startPoint.left, y: startPoint.top }; - const end = { x: start.x, y: height }; - await this.mouseDrag(start, end); + const end = { x: start.x, y: 0 }; + + await repeat(defaultScrollUntilTimes, () => + this.mouseDrag(start, end, defaultFastScrollDuration), + ); + await sleep(1000); return; } await repeat(defaultScrollUntilTimes, () => - this.mouseWheel(0, -9999999, defaultFastScrollDuration), + this.mouseWheel(0, 9999999, defaultFastScrollDuration), ); await sleep(1000); } async scrollUntilLeft(startPoint?: Point): Promise { if (startPoint) { + const { width } = await this.size(); const start = { x: startPoint.left, y: startPoint.top }; - const end = { x: 0, y: start.y }; - await this.mouseDrag(start, end); + const end = { x: width, y: start.y }; + + await repeat(defaultScrollUntilTimes, () => + this.mouseDrag(start, end, defaultFastScrollDuration), + ); + await sleep(1000); return; } await repeat(defaultScrollUntilTimes, () => - this.mouseWheel(9999999, 0, defaultFastScrollDuration), + this.mouseWheel(-9999999, 0, defaultFastScrollDuration), ); await sleep(1000); } async scrollUntilRight(startPoint?: Point): Promise { if (startPoint) { - const { width } = await this.size(); const start = { x: startPoint.left, y: startPoint.top }; - const end = { x: width, y: start.y }; - await this.mouseDrag(start, end); + const end = { x: 0, y: start.y }; + + await repeat(defaultScrollUntilTimes, () => + this.mouseDrag(start, end, defaultFastScrollDuration), + ); + await sleep(1000); return; } await repeat(defaultScrollUntilTimes, () => - this.mouseWheel(-9999999, 0, defaultFastScrollDuration), + this.mouseWheel(9999999, 0, defaultFastScrollDuration), ); await sleep(1000); } @@ -644,13 +662,13 @@ ${Object.keys(size) if (startPoint) { const start = { x: startPoint.left, y: startPoint.top }; - const endY = Math.max(0, start.y - scrollDistance); + const endY = Math.min(height, start.y + scrollDistance); const end = { x: start.x, y: endY }; await this.mouseDrag(start, end); return; } - await this.mouseWheel(0, scrollDistance); + await this.mouseWheel(0, -scrollDistance); } async scrollDown(distance?: number, startPoint?: Point): Promise { @@ -659,13 +677,13 @@ ${Object.keys(size) if (startPoint) { const start = { x: startPoint.left, y: startPoint.top }; - const endY = Math.min(height, start.y + scrollDistance); + const endY = Math.max(0, start.y - scrollDistance); const end = { x: start.x, y: endY }; await this.mouseDrag(start, end); return; } - await this.mouseWheel(0, -scrollDistance); + await this.mouseWheel(0, scrollDistance); } async scrollLeft(distance?: number, startPoint?: Point): Promise { @@ -674,13 +692,13 @@ ${Object.keys(size) if (startPoint) { const start = { x: startPoint.left, y: startPoint.top }; - const endX = Math.max(0, start.x - scrollDistance); + const endX = Math.min(width, start.x + scrollDistance); const end = { x: endX, y: start.y }; await this.mouseDrag(start, end); return; } - await this.mouseWheel(scrollDistance, 0); + await this.mouseWheel(-scrollDistance, 0); } async scrollRight(distance?: number, startPoint?: Point): Promise { @@ -689,13 +707,13 @@ ${Object.keys(size) if (startPoint) { const start = { x: startPoint.left, y: startPoint.top }; - const endX = Math.min(width, start.x + scrollDistance); + const endX = Math.max(0, start.x - scrollDistance); const end = { x: endX, y: start.y }; await this.mouseDrag(start, end); return; } - await this.mouseWheel(-scrollDistance, 0); + await this.mouseWheel(scrollDistance, 0); } private async ensureYadb() { @@ -804,6 +822,7 @@ ${Object.keys(size) private async mouseDrag( from: { x: number; y: number }, to: { x: number; y: number }, + duration?: number, ): Promise { const adb = await this.getAdb(); @@ -811,13 +830,18 @@ ${Object.keys(size) const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y); const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y); - await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`); + // Ensure duration has a default value + const swipeDuration = duration ?? 300; + + await adb.shell( + `input swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`, + ); } private async mouseWheel( deltaX: number, deltaY: number, - duration = defaultNormalScrollDuration, + duration?: number, ): Promise { const { width, height } = await this.size(); @@ -839,8 +863,11 @@ ${Object.keys(size) deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY)); // Calculate the end coordinates - const endX = startX + deltaX; - const endY = startY + deltaY; + // Note: For swipe, we need to reverse the delta direction + // because positive deltaY should scroll up (show top content), + // which requires swiping from bottom to top (decreasing Y) + const endX = startX - deltaX; + const endY = startY - deltaY; // Adjust coordinates to fit device ratio const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates( @@ -854,9 +881,12 @@ ${Object.keys(size) const adb = await this.getAdb(); + // Ensure duration has a default value + const swipeDuration = duration ?? defaultNormalScrollDuration; + // Execute the swipe operation await adb.shell( - `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`, + `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`, ); } diff --git a/packages/android/tests/ai/setting.test.ts b/packages/android/tests/ai/setting.test.ts index e8a76d8f6..edbaa0736 100644 --- a/packages/android/tests/ai/setting.test.ts +++ b/packages/android/tests/ai/setting.test.ts @@ -16,13 +16,16 @@ describe( }); await agent.launch('com.android.settings/.Settings'); - await agent.aiAction('scroll list to bottom'); await agent.aiAction('open "More settings"'); - await agent.aiAction('scroll list to bottom'); + await agent.aiAction('scroll left until left edge'); + await agent.aiAction('scroll right until right edge'); await agent.aiAction('scroll list to top'); - await agent.aiAction('swipe down one screen'); - await agent.aiAction('swipe up one screen'); + await agent.aiAction('scroll list to bottom'); + await agent.aiAction('scroll down one screen'); + await agent.aiAction('scroll up one screen'); + await agent.aiAction('scroll right one screen'); + await agent.aiAction('scroll left one screen'); }); }, 360 * 1000,