Fix dict issue for call_gpt_4_v_labeled

Author: joshbickett

operate/actions/actions.py

@@ -6,6 +6,7 @@
 import io
 import asyncio
 import aiohttp
+
 from PIL import Image
 from ultralytics import YOLO
 import google.generativeai as genai
@@ -329,22 +330,19 @@ async def call_gpt_4_v_labeled(messages, objective):
         decision_messages = messages.copy()
         decision_messages.append(decision_message)
 
-        click_future = await fetch_openai_response_async(click_messages)
-        decision_future = await fetch_openai_response_async(decision_messages)
-
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
+        click_future = fetch_openai_response_async(click_messages)
+        decision_future = fetch_openai_response_async(decision_messages)
 
-        click_response, decision_response = loop.run_until_complete(
-            asyncio.gather(click_future, decision_future)
+        click_response, decision_response = await asyncio.gather(
+            click_future, decision_future
         )
 
-        loop.close()
-
         # Extracting the message content from the ChatCompletionMessage object
-        click_content = click_response.choices[0].message.content
+        click_content = click_response.get("choices")[0].get("message").get("content")
 
-        decision_content = decision_response.choices[0].message.content
+        decision_content = (
+            decision_response.get("choices")[0].get("message").get("content")
+        )
 
         if not decision_content.startswith("CLICK"):
             return decision_content

operate/utils/action_util.py

@@ -1,9 +1,11 @@
 import pyautogui
-import platform 
+import platform
 import time
 import math
+
 from operate.utils.utils import convert_percent_to_decimal
 
+
 def keyboard_type(text):
     """
     Types the given text using the keyboard.
@@ -20,6 +22,7 @@ def keyboard_type(text):
     pyautogui.press("enter")
     return "Type: " + text
 
+
 def search(text):
     """
     Searches for a program or file by typing the given text in the search bar and pressing Enter.
@@ -49,6 +52,7 @@ def search(text):
     pyautogui.press("enter")
     return "Open program: " + text
 
+
 def click_at_percentage(
     x_percentage, y_percentage, duration=0.2, circle_radius=50, circle_duration=0.5
 ):
@@ -125,4 +129,3 @@ def get_last_assistant_message(messages):
             else:
                 return messages[index]
     return None  # Return None if no assistant message is found
-

operate/utils/utils.py

@@ -1,5 +1,7 @@
 import json
 import re
+
+
 def convert_percent_to_decimal(percent_str):
     """
     Converts a percentage string to a decimal value.
@@ -26,7 +28,6 @@ def convert_percent_to_decimal(percent_str):
     except ValueError as e:
         print(f"Error converting percent to decimal: {e}")
         return None
-    
 
 
 def extract_json_from_string(s):
@@ -52,7 +53,8 @@ def extract_json_from_string(s):
     except Exception as e:
         print(f"Error parsing JSON: {e}")
         return None
-    
+
+
 def parse_response(response):
     """
     Parses the given response and returns a dictionary with the type and data.
@@ -97,4 +99,4 @@ def parse_response(response):
             search_data = re.search(r"SEARCH (.+)", response).group(1)
         return {"type": "SEARCH", "data": search_data}
 
-    return {"type": "UNKNOWN", "data": response}
+    return {"type": "UNKNOWN", "data": response}