Better server, OS mode image fix, better profiles, Colab fix, SMS enhancement

Author: KillianLucas

interpreter/core/async_core.py

@@ -0,0 +1,35 @@
+import json
+import threading
+import time
+
+from core import OpenInterpreter
+
+
+class AsyncOpenInterpreter(OpenInterpreter):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.async_thread = None
+        self.input_queue
+        self.output_queue
+
+    async def input(self, chunk):
+        """
+        Expects a chunk in streaming LMC format.
+        """
+        try:
+            chunk = json.loads(chunk)
+        except:
+            pass
+
+        if "start" in chunk:
+            self.async_thread.join()
+        elif "end" in chunk:
+            if self.async_thread is None or not self.async_thread.is_alive():
+                self.async_thread = threading.Thread(target=self.complete)
+            self.async_thread.start()
+        else:
+            await self._add_to_queue(self._input_queue, chunk)
+
+    async def output(self, *args, **kwargs):
+        # Your async output code here
+        pass

interpreter/core/computer/contacts/contacts.py

@@ -1,73 +1,81 @@
 import platform
+
 from ..utils.run_applescript import run_applescript_capture
 
+
 class Contacts:
     def __init__(self, computer):
         self.computer = computer
 
-
     def get_phone_number(self, contact_name):
         """
         Returns the phone number of a contact by name.
         """
-        if platform.system() != 'Darwin':
+        if platform.system() != "Darwin":
             return "This method is only supported on MacOS"
-        
-        script = f'''
-        tell application "Contacts"
-            set thePerson to first person whose name is "{contact_name}"
-            set theNumber to value of first phone of thePerson
-            return theNumber
+
+        script = f"""
+        tell application "System Events" to tell process "Finder"
+            open location "addressbook://"
+            tell application "Contacts"
+                set thePerson to first person whose name is "{contact_name}"
+                if exists thePerson then
+                    set theNumber to value of first phone of thePerson
+                    return theNumber
+                else
+                    return "Contact not found"
+                end if
+            end tell
         end tell
-        '''
+        """
         stout, stderr = run_applescript_capture(script)
         # If the person is not found, we will try to find similar contacts
-        if "Can’t get person" in stderr:
-            names= self.get_full_names_from_first_name(contact_name)
-            if names == "No contacts found":
-                return "No contacts found"
+        if "Can’t get person" in stderr or not stout:
+            names = self.get_full_names_from_first_name(contact_name)
+            if "No contacts found" in names or not names:
+                raise Exception("Contact not found")
             else:
                 # Language model friendly error message
-                return f"A contact for '{contact_name}' was not found, perhaps one of these similar contacts might be what you are looking for? {names} \n Please try again and provide a more specific contact name."
+                raise Exception(
+                    f"A contact for '{contact_name}' was not found, perhaps one of these similar contacts might be what you are looking for? {names} \n Please try again and provide a more specific contact name."
+                )
         else:
-            return stout.replace('\n', '')
-
+            return stout.replace("\n", "")
 
     def get_email_address(self, contact_name):
         """
         Returns the email address of a contact by name.
         """
-        if platform.system() != 'Darwin':
+        if platform.system() != "Darwin":
             return "This method is only supported on MacOS"
-        
-        script = f'''
+
+        script = f"""
         tell application "Contacts"
             set thePerson to first person whose name is "{contact_name}"
             set theEmail to value of first email of thePerson
             return theEmail
         end tell
-        '''
+        """
         stout, stderr = run_applescript_capture(script)
         # If the person is not found, we will try to find similar contacts
         if "Can’t get person" in stderr:
-            names= self.get_full_names_from_first_name(contact_name)
+            names = self.get_full_names_from_first_name(contact_name)
             if names == "No contacts found":
                 return "No contacts found"
             else:
                 # Language model friendly error message
                 return f"A contact for '{contact_name}' was not found, perhaps one of these similar contacts might be what you are looking for? {names} \n Please try again and provide a more specific contact name."
         else:
-            return stout.replace('\n', '')
-
+            return stout.replace("\n", "")
 
     def get_full_names_from_first_name(self, first_name):
         """
         Returns a list of full names of contacts that contain the first name provided.
         """
-        if platform.system() != 'Darwin':
+        if platform.system() != "Darwin":
             return "This method is only supported on MacOS"
-        
-        script = f'''
+
+        script = f"""
         tell application "Contacts"
             set matchingPeople to every person whose name contains "{first_name}"
             set namesList to {{}}
@@ -76,10 +84,9 @@ def get_full_names_from_first_name(self, first_name):
             end repeat
             return namesList
         end tell
-        '''
+        """
         names, _ = run_applescript_capture(script)
         if names:
             return names
         else:
             return "No contacts found."
-        

interpreter/core/computer/display/display.py

@@ -10,6 +10,7 @@
 from io import BytesIO
 
 import requests
+from IPython.display import display
 from PIL import Image
 
 from ...utils.lazy_import import lazy_import
@@ -19,7 +20,11 @@
 # from utils.get_active_window import get_active_window
 
 # Lazy import of optional packages
-cv2 = lazy_import("cv2")
+try:
+    cv2 = lazy_import("cv2")
+except:
+    cv2 = None  # Fixes colab error
+
 pyautogui = lazy_import("pyautogui")
 np = lazy_import("numpy")
 plt = lazy_import("matplotlib.pyplot")
@@ -86,7 +91,6 @@ def screenshot(
         show=True,
         quadrant=None,
         active_app_only=True,
-        force_image=False,
         combine_screens=True,
     ):
         """
@@ -133,7 +137,7 @@ def screenshot(
                         )
                     )
                     message = format_to_recipient(
-                        "Taking a screenshot of the active app (recommended). To take a screenshot of the entire screen (uncommon), use computer.display.view(active_app_only=False).",
+                        "Taking a screenshot of the active app. To take a screenshot of the entire screen (uncommon), use computer.view(active_app_only=False).",
                         "assistant",
                     )
                     print(message)
@@ -145,7 +149,7 @@ def screenshot(
                     screen=screen, combine_screens=combine_screens
                 )  #  this function uses pyautogui.screenshot which works fine for all OS (mac, linux and windows)
                 message = format_to_recipient(
-                    "Taking a screenshot of the entire screen. This is not recommended. You (the language model assistant) will receive it with low resolution.\n\nTo maximize performance, use computer.display.view(active_app_only=True). This will produce an ultra high quality image of the active application.",
+                    "Taking a screenshot of the entire screen.\n\nTo focus on the active app, use computer.view(active_app_only=True).",
                     "assistant",
                 )
                 print(message)
@@ -181,18 +185,12 @@ def screenshot(
             screenshot = screenshot.convert("RGB")
 
         if show:
-            # Show the image using matplotlib
+            # Show the image using IPython display
             if isinstance(screenshot, list):
                 for img in screenshot:
-                    plt.imshow(np.array(img))
-                    plt.show()
+                    display(img)
             else:
-                plt.imshow(np.array(screenshot))
-
-            with warnings.catch_warnings():
-                # It displays an annoying message about Agg not being able to display something or WHATEVER
-                warnings.simplefilter("ignore")
-                plt.show()
+                display(screenshot)
 
         return screenshot  # this will be a list of combine_screens == False
 

interpreter/core/computer/mouse/mouse.py

@@ -1,13 +1,17 @@
 import time
 import warnings
 
+from IPython.display import display
+from PIL import Image
+
 from ...utils.lazy_import import lazy_import
 from ..utils.recipient_utils import format_to_recipient
 
 # Lazy import of optional packages
-cv2 = lazy_import(
-    "cv2",
-)
+try:
+    cv2 = lazy_import("cv2")
+except:
+    cv2 = None  # Fixes colab error
 np = lazy_import("numpy")
 pyautogui = lazy_import("pyautogui")
 plt = lazy_import("matplotlib.pyplot")
@@ -103,10 +107,8 @@ def move(self, *args, x=None, y=None, icon=None, text=None, screenshot=None):
                             cv2.LINE_AA,
                         )
 
-                    plt.imshow(img_draw)
-                    with warnings.catch_warnings():
-                        warnings.simplefilter("ignore")
-                        plt.show()
+                    img_pil = Image.fromarray(img_draw)
+                    display(img_pil)
 
                 coordinates = [
                     f"{i}: ({int(item['coordinates'][0]*self.computer.display.width)}, {int(item['coordinates'][1]*self.computer.display.height)}) "

interpreter/core/computer/sms/sms.py

@@ -1,37 +1,109 @@
+import datetime
+import os
+import plistlib
+import sqlite3
 import subprocess
-import platform
-from ..utils.run_applescript import run_applescript
+import time
 
 
 class SMS:
     def __init__(self, computer):
         self.computer = computer
-        self.messages_app = "Messages"
-
+        self.database_path = self.resolve_database_path()
 
+    def resolve_database_path(self):
+        if os.geteuid() == 0:  # Running as root
+            home_directory = os.path.expanduser(f"~{os.environ.get('SUDO_USER')}")
+        else:
+            home_directory = os.path.expanduser("~")
+        return f"{home_directory}/Library/Messages/chat.db"
 
     def send(self, to, message):
+        message_escaped = message.replace('"', '\\"').replace("\\", "\\\\")
+        script = f"""
+        tell application "Messages"
+            set targetBuddy to "{to}"
+            send "{message_escaped}" to buddy targetBuddy of (service 1 whose service type is iMessage)
+        end tell
         """
-        Sends an SMS message to the specified recipient using the Messages app.
-        """
-        # Check if the operating system is MacOS, as this functionality is MacOS-specific.
-        if platform.system() != 'Darwin':
-            return "This method is only supported on MacOS"
-        
-        # Remove any newline characters from the recipient number.
-        to = to.replace("\n", "")
-        # Escape double quotes in the message and recipient variables to prevent script errors.
-        escaped_message = message.replace('"', '\\"')
-        escaped_to = to.replace('"', '\\"')
+        subprocess.run(["osascript", "-e", script], check=True)
+        return "Message sent successfully"
 
-        script = f"""
-        tell application "Messages"                                                   
-            set targetBuddy to buddy "{escaped_to}" of service 1                        
-            send "{escaped_message}" to targetBuddy                                
-        end tell 
+    def get(self, contact=None, limit=10, substring=None):
+        if not self.can_access_database():
+            self.prompt_full_disk_access()
+
+        conn = sqlite3.connect(self.database_path)
+        conn.row_factory = sqlite3.Row  # Set row factory
+        cursor = conn.cursor()
+        query = """
+SELECT message.*, handle.id as sender FROM message
+LEFT JOIN handle ON message.handle_id = handle.ROWID
         """
+        params = []
+        conditions = []
+
+        if contact:
+            conditions.append("handle.id=?")
+            params.append(contact)
+        if substring:
+            conditions.append("message.text LIKE ?")
+            params.append(f"%{substring}%")
+        if conditions:
+            query += " WHERE " + " AND ".join(conditions)
+        query += " ORDER BY message.date DESC"
+
+        cursor.execute(query, params)
+
+        # Parse plist data and make messages readable
+        readable_messages = []
+        while len(readable_messages) < limit:
+            try:
+                message = cursor.fetchone()
+                if message is None:
+                    break
+                message_dict = dict(message)  # Convert row to dictionary
+                text_data = message_dict.get("text")
+                if text_data:
+                    try:
+                        # Try to parse as plist
+                        plist_data = plistlib.loads(text_data)
+                        text = plist_data.get("NS.string", "")
+                    except:
+                        # If plist parsing fails, use the raw string
+                        text = text_data
+                    if text:  # Only add messages with content
+                        # Convert Apple timestamp to datetime
+                        date = datetime.datetime(2001, 1, 1) + datetime.timedelta(
+                            seconds=message_dict.get("date") / 10**9
+                        )
+                        sender = message_dict.get("sender")
+                        if message_dict.get("is_from_me") == 1:
+                            sender = "(Me)"
+                        readable_messages.append(
+                            {"date": date, "from": sender, "text": text}
+                        )
+            except sqlite3.Error as e:
+                break
+
+        conn.close()
+        return readable_messages
+
+    def can_access_database(self):
         try:
-            run_applescript(script)
-            return "SMS message sent"
-        except subprocess.CalledProcessError:
-            return "An error occurred while sending the SMS. Please check the recipient number and try again."
+            with open(self.database_path, "r"):
+                return True
+        except IOError:
+            return False
+
+    def prompt_full_disk_access(self):
+        script = """
+        tell application "System Preferences"
+            activate
+        end tell
+        delay 1
+        tell application "System Events"
+            display dialog "This application requires Full Disk Access to function properly.\\n\\nPlease follow these steps:\\n1. Open the Security & Privacy panel.\\n2. Go to the Full Disk Access section.\\n3. Click the lock icon and enter your password to make changes.\\n4. Click the '+' button and add your terminal application (e.g., Terminal, iTerm).\\n5. Restart the application after granting access." buttons {"OK"} default button "OK"
+        end tell
+        """
+        subprocess.run(["osascript", "-e", script], check=True)