Fixing discussion object when adding images w/o detail (#128)

* switching to goal_object in xray

* adding excpetion for when summary isnt available

* fixing discussion w/ images

Author: TLSDC

src/agentlab/analyze/agent_xray.py

@@ -21,6 +21,7 @@
 from agentlab.experiments.exp_utils import RESULTS_DIR
 from agentlab.experiments.study import get_most_recent_study
 from agentlab.llm.chat_api import make_system_message, make_user_message
+from agentlab.llm.llm_utils import BaseMessage as AgentLabBaseMessage
 from agentlab.llm.llm_utils import Discussion
 
 select_dir_instructions = "Select Experiment Directory"
@@ -740,7 +741,7 @@ def get_episode_info(info: Info):
         steps_info = info.exp_result.steps_info
         step_info = steps_info[info.step]
         try:
-            goal = step_info.obs["goal"]
+            goal = step_info.obs["goal_object"]
         except KeyError:
             goal = None
         try:
@@ -757,7 +758,7 @@ def get_episode_info(info: Info):
 
 **Goal:**
 
-{code(goal)}
+{code(str(AgentLabBaseMessage('', goal)))}
 
 **Task info:**
 
@@ -992,20 +993,23 @@ def get_directory_contents(results_dir: Path):
             continue
 
         exp_description = dir.name
-        # get summary*.csv files and find the most recent
-        summary_files = list(dir.glob("summary*.csv"))
-        if len(summary_files) != 0:
-            most_recent_summary = max(summary_files, key=os.path.getctime)
-            summary_df = pd.read_csv(most_recent_summary)
-
-            # get row with max avg_reward
-            max_reward_row = summary_df.loc[summary_df["avg_reward"].idxmax()]
-            reward = max_reward_row["avg_reward"] * 100
-            completed = max_reward_row["n_completed"]
-            n_err = max_reward_row["n_err"]
-            exp_description += (
-                f" - avg-reward: {reward:.1f}% - completed: {completed} - errors: {n_err}"
-            )
+        try:
+            # get summary*.csv files and find the most recent
+            summary_files = list(dir.glob("summary*.csv"))
+            if len(summary_files) != 0:
+                most_recent_summary = max(summary_files, key=os.path.getctime)
+                summary_df = pd.read_csv(most_recent_summary)
+
+                # get row with max avg_reward
+                max_reward_row = summary_df.loc[summary_df["avg_reward"].idxmax()]
+                reward = max_reward_row["avg_reward"] * 100
+                completed = max_reward_row["n_completed"]
+                n_err = max_reward_row["n_err"]
+                exp_description += (
+                    f" - avg-reward: {reward:.1f}% - completed: {completed} - errors: {n_err}"
+                )
+        except Exception as e:
+            print(f"Error while reading summary file: {e}")
 
         exp_descriptions.append(exp_description)
 

src/agentlab/llm/llm_utils.py

@@ -8,8 +8,7 @@
 import time
 from copy import deepcopy
 from functools import cache
-from typing import TYPE_CHECKING
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
 from warnings import warn
 
 import numpy as np
@@ -356,7 +355,7 @@ def add_image(self, image: np.ndarray | Image.Image | str, detail: str = None):
         if detail:
             self.add_content("image_url", {"url": image_url, "detail": detail})
         else:
-            self.add_content("image_url", image_url)
+            self.add_content("image_url", {"url": image_url})
 
     def to_markdown(self):
         if isinstance(self["content"], str):