fix(computer-use): scale scroll amounts (#23)

* fix(computer-use): scale scroll amounts in TypeScript

* fix(computer-use): python template scrolling scale

* fix(computer-use): print scroll distance in pixels in TypeScript

---------

Co-authored-by: Matt Marangoni <[email protected]>

Author: matthewjmarangoni

templates/python/computer-use/tools/computer.py

@@ -322,16 +322,26 @@ async def __call__(
                 await self.page.mouse.move(x, y)
 
             # Map scroll directions to Playwright's wheel events
+            page_dimensions = await self.page.evaluate(
+                "() => Promise.resolve({ h: window.innerHeight, w: window.innerWidth })"
+            )
+            page_partitions = 25
+            scroll_factor = scroll_amount / page_partitions
+            page_width = page_dimensions['w']
+            page_height = page_dimensions['h']
+
             delta_x = 0
             delta_y = 0
             if scroll_direction == "up":
-                delta_y = -scroll_amount * 100
+                delta_y = -scroll_factor * page_height
             elif scroll_direction == "down":
-                delta_y = scroll_amount * 100
+                delta_y = scroll_factor * page_height
             elif scroll_direction == "left":
-                delta_x = -scroll_amount * 100
+                delta_x = -scroll_factor * page_width
             elif scroll_direction == "right":
-                delta_x = scroll_amount * 100
+                delta_x = scroll_factor * page_width
+
+            print(f"Scrolling {abs(delta_x) if delta_x != 0 else abs(delta_y):.02f} pixels {scroll_direction}")
 
             await self.page.mouse.wheel(delta_x=delta_x, delta_y=delta_y)
             return await self.screenshot()
@@ -391,4 +401,4 @@ async def __call__(
 
         return await super().__call__(
             action=action, text=text, coordinate=coordinate, key=key, **kwargs
-        )
+        )

templates/typescript/computer-use/tools/computer.ts

@@ -190,11 +190,19 @@ export class ComputerTool implements BaseAnthropicTool {
         await this.page.waitForTimeout(100);
       }
 
-      const amount = scrollAmountValue || 100;
+      const pageDimensions = await this.page.evaluate(() => {
+	      return { h: window.innerHeight, w: window.innerWidth };
+      });
+      const pagePartitions = 25;
+      const scrollFactor = (scrollAmountValue || 10) / pagePartitions;
 
       if (scrollDirection === 'down' || scrollDirection === 'up') {
+        const amount = pageDimensions.h * scrollFactor;
+        console.log(`Scrolling ${amount.toFixed(2)} pixels ${scrollDirection}`);
         await this.page.mouse.wheel(0, scrollDirection === 'down' ? amount : -amount);
       } else {
+        const amount = pageDimensions.w * scrollFactor;
+        console.log(`Scrolling ${amount.toFixed(2)} pixels ${scrollDirection}`);
         await this.page.mouse.wheel(scrollDirection === 'right' ? amount : -amount, 0);
       }