Skip to content

fix(android): simplify scroll methods by removing startPoint parameter and enhancing scrolling logic #1039

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 13, 2025
Merged

fix(android): simplify scroll methods by removing startPoint parameter and enhancing scrolling logic #1039

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70ffb6a
fix(android): simplify scroll methods by removing startPoint paramete…
quanru Aug 12, 2025
2a31b74
fix(android): enhance scrolling methods to support startPoint paramet…
quanru Aug 13, 2025
026cf8b
docs(site): update acknowledgments to include Rslib as a build tool
quanru Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 61 additions & 31 deletions packages/android/src/page/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -521,11 +521,14 @@ ${Object.keys(size)
get mouse() {
return {
click: (x: number, y: number) => this.mouseClick(x, y),
wheel: (deltaX: number, deltaY: number) =>
this.mouseWheel(deltaX, deltaY),
wheel: (deltaX: number, deltaY: number, duration?: number) =>
this.mouseWheel(deltaX, deltaY, duration),
move: (x: number, y: number) => this.mouseMove(x, y),
drag: (from: { x: number; y: number }, to: { x: number; y: number }) =>
this.mouseDrag(from, to),
drag: (
from: { x: number; y: number },
to: { x: number; y: number },
duration?: number,
) => this.mouseDrag(from, to, duration),
};
}

Expand Down Expand Up @@ -581,59 +584,74 @@ ${Object.keys(size)

async scrollUntilTop(startPoint?: Point): Promise<void> {
if (startPoint) {
const { height } = await this.size();
const start = { x: startPoint.left, y: startPoint.top };
const end = { x: start.x, y: 0 };
const end = { x: start.x, y: height };

await this.mouseDrag(start, end);
await repeat(defaultScrollUntilTimes, () =>
this.mouseDrag(start, end, defaultFastScrollDuration),
);
await sleep(1000);
return;
}

await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(0, 9999999, defaultFastScrollDuration),
this.mouseWheel(0, -9999999, defaultFastScrollDuration),
);
await sleep(1000);
}

async scrollUntilBottom(startPoint?: Point): Promise<void> {
if (startPoint) {
const { height } = await this.size();
const start = { x: startPoint.left, y: startPoint.top };
const end = { x: start.x, y: height };
await this.mouseDrag(start, end);
const end = { x: start.x, y: 0 };

await repeat(defaultScrollUntilTimes, () =>
this.mouseDrag(start, end, defaultFastScrollDuration),
);
await sleep(1000);
return;
}

await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(0, -9999999, defaultFastScrollDuration),
this.mouseWheel(0, 9999999, defaultFastScrollDuration),
);
await sleep(1000);
}

async scrollUntilLeft(startPoint?: Point): Promise<void> {
if (startPoint) {
const { width } = await this.size();
const start = { x: startPoint.left, y: startPoint.top };
const end = { x: 0, y: start.y };
await this.mouseDrag(start, end);
const end = { x: width, y: start.y };

await repeat(defaultScrollUntilTimes, () =>
this.mouseDrag(start, end, defaultFastScrollDuration),
);
await sleep(1000);
return;
}

await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(9999999, 0, defaultFastScrollDuration),
this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
);
await sleep(1000);
}

async scrollUntilRight(startPoint?: Point): Promise<void> {
if (startPoint) {
const { width } = await this.size();
const start = { x: startPoint.left, y: startPoint.top };
const end = { x: width, y: start.y };
await this.mouseDrag(start, end);
const end = { x: 0, y: start.y };

await repeat(defaultScrollUntilTimes, () =>
this.mouseDrag(start, end, defaultFastScrollDuration),
);
await sleep(1000);
return;
}

await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
this.mouseWheel(9999999, 0, defaultFastScrollDuration),
);
await sleep(1000);
}
Expand All @@ -644,13 +662,13 @@ ${Object.keys(size)

if (startPoint) {
const start = { x: startPoint.left, y: startPoint.top };
const endY = Math.max(0, start.y - scrollDistance);
const endY = Math.min(height, start.y + scrollDistance);
const end = { x: start.x, y: endY };
await this.mouseDrag(start, end);
return;
}

await this.mouseWheel(0, scrollDistance);
await this.mouseWheel(0, -scrollDistance);
}

async scrollDown(distance?: number, startPoint?: Point): Promise<void> {
Expand All @@ -659,13 +677,13 @@ ${Object.keys(size)

if (startPoint) {
const start = { x: startPoint.left, y: startPoint.top };
const endY = Math.min(height, start.y + scrollDistance);
const endY = Math.max(0, start.y - scrollDistance);
const end = { x: start.x, y: endY };
await this.mouseDrag(start, end);
return;
}

await this.mouseWheel(0, -scrollDistance);
await this.mouseWheel(0, scrollDistance);
}

async scrollLeft(distance?: number, startPoint?: Point): Promise<void> {
Expand All @@ -674,13 +692,13 @@ ${Object.keys(size)

if (startPoint) {
const start = { x: startPoint.left, y: startPoint.top };
const endX = Math.max(0, start.x - scrollDistance);
const endX = Math.min(width, start.x + scrollDistance);
const end = { x: endX, y: start.y };
await this.mouseDrag(start, end);
return;
}

await this.mouseWheel(scrollDistance, 0);
await this.mouseWheel(-scrollDistance, 0);
}

async scrollRight(distance?: number, startPoint?: Point): Promise<void> {
Expand All @@ -689,13 +707,13 @@ ${Object.keys(size)

if (startPoint) {
const start = { x: startPoint.left, y: startPoint.top };
const endX = Math.min(width, start.x + scrollDistance);
const endX = Math.max(0, start.x - scrollDistance);
const end = { x: endX, y: start.y };
await this.mouseDrag(start, end);
return;
}

await this.mouseWheel(-scrollDistance, 0);
await this.mouseWheel(scrollDistance, 0);
}

private async ensureYadb() {
Expand Down Expand Up @@ -804,20 +822,26 @@ ${Object.keys(size)
private async mouseDrag(
from: { x: number; y: number },
to: { x: number; y: number },
duration?: number,
): Promise<void> {
const adb = await this.getAdb();

// Use adjusted coordinates
const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);

await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
// Ensure duration has a default value
const swipeDuration = duration ?? 300;

await adb.shell(
`input swipe ${fromX} ${fromY} ${toX} ${toY} ${swipeDuration}`,
);
}

private async mouseWheel(
deltaX: number,
deltaY: number,
duration = defaultNormalScrollDuration,
duration?: number,
): Promise<void> {
const { width, height } = await this.size();

Expand All @@ -839,8 +863,11 @@ ${Object.keys(size)
deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));

// Calculate the end coordinates
const endX = startX + deltaX;
const endY = startY + deltaY;
// Note: For swipe, we need to reverse the delta direction
// because positive deltaY should scroll up (show top content),
// which requires swiping from bottom to top (decreasing Y)
const endX = startX - deltaX;
const endY = startY - deltaY;

// Adjust coordinates to fit device ratio
const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
Expand All @@ -854,9 +881,12 @@ ${Object.keys(size)

const adb = await this.getAdb();

// Ensure duration has a default value
const swipeDuration = duration ?? defaultNormalScrollDuration;

// Execute the swipe operation
await adb.shell(
`input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`,
`input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${swipeDuration}`,
);
}

Expand Down
11 changes: 7 additions & 4 deletions packages/android/tests/ai/setting.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ describe(
});

await agent.launch('com.android.settings/.Settings');

await agent.aiAction('scroll list to bottom');
await agent.aiAction('open "More settings"');
await agent.aiAction('scroll list to bottom');
await agent.aiAction('scroll left until left edge');
await agent.aiAction('scroll right until right edge');
await agent.aiAction('scroll list to top');
await agent.aiAction('swipe down one screen');
await agent.aiAction('swipe up one screen');
await agent.aiAction('scroll list to bottom');
await agent.aiAction('scroll down one screen');
await agent.aiAction('scroll up one screen');
await agent.aiAction('scroll right one screen');
await agent.aiAction('scroll left one screen');
});
},
360 * 1000,
Expand Down