wip

Author: abrichr

experiments/describe_action.py

@@ -0,0 +1,22 @@
+from pprint import pformat
+
+from loguru import logger
+
+from openadapt.db import crud
+
+
+def main() -> None:
+    session = crud.get_new_session(read_only=True)
+    recording = crud.get_latest_recording(session)
+    action_events = recording.processed_action_events
+    descriptions = []
+    for action in action_events:
+        logger.info(f"{action=}")
+        description = action.prompt_for_description()
+        logger.info(f"{description=}")
+        descriptions.append(description)
+    logger.info(f"descriptions=\n{pformat(descriptions)}")
+
+
+if __name__ == "__main__":
+    main()

openadapt/drivers/anthropic.py

@@ -5,7 +5,7 @@
 from PIL import Image
 import anthropic
 
-from openadapt import cache, utils
+from openadapt import cache
 from openadapt.config import config
 from openadapt.custom_logger import logger
 
@@ -148,3 +148,7 @@ def prompt(
     result = get_completion(payload)
     pprint(f"result=\n{result}")  # Log result for debugging
     return result
+
+
+# avoid circular import
+from openadapt import utils  # noqa

openadapt/models.py

@@ -16,6 +16,7 @@
 
 from openadapt.config import config
 from openadapt.custom_logger import logger
+from openadapt.drivers import anthropic
 from openadapt.db import db
 from openadapt.privacy.base import ScrubbingProvider, TextScrubbingMixin
 from openadapt.privacy.providers import ScrubProvider
@@ -544,6 +545,56 @@ def next_event(self) -> Union["ActionEvent", None]:
 
         return None
 
+    def prompt_for_description(self) -> str:
+        """Use the Anthropic API to describe what is happening in the action event.
+
+        Returns:
+            str: The description of the action event.
+        """
+        # Collect the relevant information for the prompt
+        prompt_details = {
+            "name": self.name,
+            "mouse_x": self.mouse_x,
+            "mouse_y": self.mouse_y,
+            "mouse_dx": self.mouse_dx,
+            "mouse_dy": self.mouse_dy,
+            "text": self.text,
+        }
+
+        # Convert the details to a readable format for the prompt
+        action_text = "\n".join(
+            [
+                f"{key}: {value}"
+                for key, value in prompt_details.items() if value is not None
+            ]
+        )
+        prompt_text = f"Action:\n{action_text}"
+
+        # Add contextual instructions for the prompt
+        system_prompt = (
+            "You are an assistant tasked with describing user interactions in a graphical interface. "
+            "Based on the following Action details and the provided screenshot, describe the action taking place "
+            "as clearly as possible. Do not describe the screenshot in its entirety, only what is relevant for describing the action."
+        )
+
+        logger.info(f"system_prompt=\n{system_prompt}")
+        logger.info(f"prompt_text=\n{prompt_text}")
+
+        # Call the Anthropic API
+        try:
+            description = anthropic.prompt(
+                prompt=prompt_text,
+                system_prompt=system_prompt,
+                images=[self.screenshot.image],
+                max_tokens=256,  # Adjust token limit as necessary
+            )
+        except Exception as e:
+            logger.exception(f"Error while prompting for description: {e}")
+            description = "An error occurred while generating the description."
+
+        return description
+
+
 
 class WindowEvent(db.Base):
     """Class representing a window event in the database."""

openadapt/plotting.py

@@ -12,7 +12,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-from openadapt import common, contrib, models, utils
+from openadapt import common, models, utils
 from openadapt.config import PERFORMANCE_PLOTS_DIR_PATH, config
 from openadapt.custom_logger import logger
 from openadapt.models import ActionEvent
@@ -792,6 +792,8 @@ def get_marked_image(
         Image.Image: The marked image, where marks and/or masks are applied based on
         the specified confidence and IoU thresholds and the include flags.
     """
+    from openadapt import contrib
+
     image_arr = np.asarray(original_image)
 
     # The rest of this function is copied from