Merge branch 'aj/tool_use_agent_chat_completion_support' into allac/next-agent

Author: recursix

src/agentlab/agents/agent_utils.py

@@ -1,5 +1,7 @@
 from logging import warning
+from typing import Optional, Tuple
 
+import numpy as np
 from PIL import Image, ImageDraw
 from playwright.sync_api import Page
 
@@ -42,10 +44,37 @@ def tag_screenshot_with_action(screenshot: Image, action: str) -> Image:
             )
         except (ValueError, IndexError) as e:
             warning(f"Failed to parse action '{action}': {e}")
+
+    elif action.startswith("mouse_drag_and_drop"):
+        try:
+            func_name, parsed_args = parse_func_call_string(action)
+            if func_name == "mouse_drag_and_drop" and parsed_args is not None:
+                args, kwargs = parsed_args
+                x1, y1, x2, y2 = None, None, None, None
+
+                if args and len(args) >= 4:
+                    # Positional arguments: mouse_drag_and_drop(x1, y1, x2, y2)
+                    x1, y1, x2, y2 = map(float, args[:4])
+                elif kwargs:
+                    # Keyword arguments: mouse_drag_and_drop(from_x=x1, from_y=y1, to_x=x2, to_y=y2)
+                    x1 = float(kwargs.get("from_x", 0))
+                    y1 = float(kwargs.get("from_y", 0))
+                    x2 = float(kwargs.get("to_x", 0))
+                    y2 = float(kwargs.get("to_y", 0))
+
+                if all(coord is not None for coord in [x1, y1, x2, y2]):
+                    draw = ImageDraw.Draw(screenshot)
+                    # Draw the main line
+                    draw.line((x1, y1, x2, y2), fill="red", width=2)
+                    # Draw arrowhead at the end point using the helper function
+                    draw_arrowhead(draw, (x1, y1), (x2, y2))
+        except (ValueError, IndexError) as e:
+            warning(f"Failed to parse action '{action}': {e}")
     return screenshot
 
 
 def add_mouse_pointer_from_action(screenshot: Image, action: str) -> Image.Image:
+
     if action.startswith("mouse_click"):
         try:
             coords = action[action.index("(") + 1 : action.index(")")].split(",")
@@ -85,6 +114,23 @@ def draw_mouse_pointer(image: Image.Image, x: int, y: int) -> Image.Image:
     return Image.alpha_composite(image.convert("RGBA"), overlay)
 
 
+def draw_arrowhead(draw, start, end, arrow_length=15, arrow_angle=30):
+    from math import atan2, cos, radians, sin
+
+    angle = atan2(end[1] - start[1], end[0] - start[0])
+    left = (
+        end[0] - arrow_length * cos(angle - radians(arrow_angle)),
+        end[1] - arrow_length * sin(angle - radians(arrow_angle)),
+    )
+    right = (
+        end[0] - arrow_length * cos(angle + radians(arrow_angle)),
+        end[1] - arrow_length * sin(angle + radians(arrow_angle)),
+    )
+    draw.line([end, left], fill="red", width=4)
+    draw.line([end, right], fill="red", width=4)
+
+
+
 def draw_click_indicator(image: Image.Image, x: int, y: int) -> Image.Image:
     """
     Draws a click indicator (+ shape with disconnected lines) at (x, y) on the image.

src/agentlab/agents/tool_use_agent/hint_db.csv

@@ -4,4 +4,12 @@ June 4,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-
 June 4,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,allac,Filling up form,GUI elements surrounded by a red rectangle often means there is an error in the content
 June 4,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,allac,Search results,The scroll bar indicates that there is more than 1 flights available in the search. Make sure to select the one matching the task goal among all possible flights. 
 June 7,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,allac,Filling up form,"If you suspect an error in an element of a form, ""ControlOrMeta+a"" to select all and overwrite the content"
-June 9,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,allac,Filling up form,"The main mistake is to type in the wrong field. Make sure the correct field is activated by clicking into it and seeing it activated with a blue rectangle before proceeding with writing. Otherwise, it will append to the currently activated field."
+June 9,miniwob.book-flight,2,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,allac,Filling up form,"The main mistake is to type in the wrong field. Make sure the correct field is activated by clicking into it and seeing it activated with a blue rectangle before proceeding with writing. Otherwise, it will append to the currently activated field."
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"For dragging tasks, use only mouse_down, move_mouse, and mouse_up. Do not perform a single continuous drag. Instead, move the mouse in steps using move_mouse, and then release with mouse_up when the target is reached."
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Avoid dragging items outside the grid. Ensure the drop coordinates are within valid grid boundaries, or the task will fail."
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Grid items are reactive: dragging an item over another may cause them to swap. Only perform mouse_down when the dragged item is correctly positioned. Intermediate movement helps trigger these reactive behaviors reliably."
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Use intermediate positions when moving items. This often leads to more stable behavior than dragging directly to the final target."
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Hovering near a target item can prompt it to move to an adjacent grid cell. Use this behavior to clear the destination before dropping the dragged item"
+June 11,miniwob.drag-items-grid,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Move items along slightly curved instead of straight lines to better imitate human-like dragging behavior."
+June 11,miniwob.drag-items,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"Move items along slightly curved instead of straight lines to better imitate human-like dragging behavior."
+June 11,miniwob.drag-items,30,claude-3-7-sonnet-20250219,MultiToolUse-claude-3-7-sonnet-20250219,miniwob,miniwob,aman,Dragging items,"For dragging tasks, use only mouse_down, move_mouse, and mouse_up. Do not perform a single continuous drag. Instead, move the mouse in steps using move_mouse, and then release with mouse_up when the target is reached."

src/agentlab/agents/tool_use_agent/multi_tool_agent.py

@@ -72,7 +72,7 @@ class StructuredDiscussion:
 
     def __init__(self, keep_last_n_obs=None):
         self.groups: list[MsgGroup] = []
-        self.keep_last_n_obs = keep_last_n_obs
+        self.keep_last_n_obs: int | None = keep_last_n_obs
 
     def append(self, message: MessageBuilder):
         """Append a message to the last group."""
@@ -96,7 +96,9 @@ def flatten(self) -> list[MessageBuilder]:
                 messages.append(group.summary)
             else:
                 messages.extend(group.messages)
-
+            # Mark all summarized messages for caching
+            if i == len(self.groups) - keep_last_n_obs:
+                messages[i].mark_all_previous_msg_for_caching()
         return messages
 
     def set_last_summary(self, summary: MessageBuilder):
@@ -452,7 +454,8 @@ def get_action(self, obs: Any) -> float:
             messages=messages,
             tool_choice="any",
             cache_tool_definition=True,
-            cache_complete_prompt=True,
+            cache_complete_prompt=False,
+            use_cache_breakpoints=True,
         )
 
         action = response.action

src/agentlab/llm/response_api.py

@@ -107,6 +107,11 @@ def add_image_url(self, image_url: str) -> "MessageBuilder":
         self.content.append({"image": image_to_png_base64_url(image_url)})
         return self
 
+    def mark_all_previous_msg_for_caching(self):
+        """Insert a cache breakpoint in the message content."""
+        # This is a placeholder for future implementation.
+        raise NotImplementedError
+
 
 # TODO: Support parallel tool calls.
 
@@ -216,6 +221,10 @@ def transform_content(self, content: ContentItem) -> ContentItem:
         else:
             raise ValueError(f"Unsupported content type: {content}")
 
+    def mark_all_previous_msg_for_caching(self) -> List[Message]:
+        """Insert a cache breakpoint in the message content to mark all previous messages for caching."""
+        self._cache_breakpoint = True
+
 
 class OpenAIChatCompletionAPIMessageBuilder(MessageBuilder):
 
@@ -521,7 +530,10 @@ def _call_api(
         sys_msg, other_msgs = self.filter_system_messages(messages)
         sys_msg_text = "\n".join(c["text"] for m in sys_msg for c in m.content)
         for msg in other_msgs:
-            input.extend(msg.prepare_message() if isinstance(msg, MessageBuilder) else [msg])
+            temp = msg.prepare_message() if isinstance(msg, MessageBuilder) else [msg]
+            if kwargs.pop("use_cache_breakpoints", False):
+                temp = self.apply_cache_breakpoints(msg, temp)
+            input.extend(temp)
 
         api_params: Dict[str, Any] = {
             "model": self.model_name,
@@ -581,6 +593,16 @@ def _parse_response(self, response: dict) -> dict:
                 result.think += output.text
         return result
 
+    # def ensure_cache_conditions(self, msgs: List[Message]) -> bool:
+    #     """Ensure API specific cache conditions are met."""
+    #     assert sum(getattr(msg, "_cache_breakpoint", 0) for msg in msgs) <= 4, "Too many cache breakpoints in the message."
+
+    def apply_cache_breakpoints(self, msg: Message, prepared_msg: dict) -> List[Message]:
+        """Apply cache breakpoints to the messages."""
+        if getattr(msg, "_cache_breakpoint", False):
+                prepared_msg[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
+        return prepared_msg
+
 
 def cua_response_to_text(action):
     """