Merge branch 'main' of https://github.com/KillianLucas/open-interpreter

Author: KillianLucas

README.md

@@ -364,6 +364,34 @@ Open Interpreter equips a [function-calling language model](https://platform.ope
 
 We then stream the model's messages, code, and your system's outputs to the terminal as Markdown.
 
+# Access Documentation Offline
+
+The full [documentation](https://docs.openinterpreter.com/) is accessible on-the-go without the need for an internet connection.
+
+[Node](https://nodejs.org/en) is a pre-requisite:
+
+- Version 18.17.0 or any later 18.x.x version.
+- Version 20.3.0 or any later 20.x.x version.
+- Any version starting from 21.0.0 onwards, with no upper limit specified.
+
+Install [Mintlify](https://mintlify.com/):
+
+```bash
+npm i -g mintlify@latest
+```
+
+Change into the docs directory and run the appropriate command:
+
+```bash
+# Assuming you're at the project's root directory
+cd ./docs
+
+# Run the documentation server
+mintlify dev
+```
+
+A new browser window should open. The documentation will be available at [http://localhost:3000](http://localhost:3000) as long as the documentation server is running.
+
 # Contributing
 
 Thank you for your interest in contributing! We welcome involvement from the community.

docs/telemetry/telemetry.mdx

@@ -12,11 +12,11 @@ If you prefer to opt out of telemetry, you can do this in two ways.
 
 ### Python
 
-Set `anonymized_telemetry` to `false` on the `interpreter` object:
+Set `disable_telemetry` to `true` on the `interpreter` object:
 
 ```python
 from interpreter import interpreter
-interpreter.anonymized_telemetry = False
+interpreter.disable_telemetry = True
 ```
 
 ### Terminal
@@ -29,20 +29,20 @@ interpreter --disable_telemetry
 
 ### Configuration File
 
-Set `anonymized_telemetry` to `false`. This will persist to future terminal sessions:
+Set `disable_telemetry` to `true`. This will persist to future terminal sessions:
 
 ```yaml
-anonymized_telemetry: false
+disable_telemetry: true
 ```
 
 ### Environment Variables
 
-Set `ANONYMIZED_TELEMETRY` to `False` in your shell or server environment.
+Set `DISABLE_TELEMETRY` to `true` in your shell or server environment.
 
 If you are running Open Interpreter on your local computer with `docker-compose` you can set this value in an `.env` file placed in the same directory as the `docker-compose.yml` file:
 
 ```
-ANONYMIZED_TELEMETRY=False
+DISABLE_TELEMETRY=true
 ```
 
 # What do you track?

interpreter/core/computer/computer.py

@@ -49,6 +49,9 @@ def __init__(self, interpreter):
         self.import_computer_api = False  # Defaults to false
         self._has_imported_computer_api = False  # Because we only want to do this once
 
+        self.import_skills = False
+        self._has_imported_skills = False
+
     # Shortcut for computer.terminal.languages
     @property
     def languages(self):

interpreter/core/computer/display/display.py

@@ -6,11 +6,15 @@
 import warnings
 from contextlib import redirect_stdout
 from io import BytesIO
-
+import io
+import subprocess
+from PIL import Image
 import requests
-
 from ...utils.lazy_import import lazy_import
 from ..utils.recipient_utils import format_to_recipient
+import cv2
+from screeninfo import get_monitors # for getting info about connected monitors
+
 
 # Still experimenting with this
 # from utils.get_active_window import get_active_window
@@ -20,6 +24,7 @@
 np = lazy_import("numpy")
 plt = lazy_import("matplotlib.pyplot")
 
+
 from ..utils.computer_vision import find_text_in_image, pytesseract_get_text
 
 
@@ -56,20 +61,30 @@ def center(self):
         """
         return self.width // 2, self.height // 2
 
-    def view(self, show=True, quadrant=None):
+    def info(self):
+        """
+        Returns a list of all connected montitor/displays and thir information
+        """
+        return get_displays()
+    
+    
+    def view(self, show=True, quadrant=None, screen=0, combine_screens=True
+    ):
         """
         Redirects to self.screenshot
         """
-        return self.screenshot(show, quadrant)
+        return self.screenshot(screen=screen, show=show, quadrant=quadrant, combine_screens=combine_screens)
 
     # def get_active_window(self):
     #     return get_active_window()
 
     def screenshot(
-        self, show=True, quadrant=None, active_app_only=False, force_image=False
+        self, screen=0, show=True, quadrant=None, active_app_only=False, force_image=False,combine_screens=True
     ):
         """
         Shows you what's on the screen by taking a screenshot of the entire screen or a specified quadrant. Returns a `pil_image` `in case you need it (rarely). **You almost always want to do this first!**
+        :param screen: specify which display; 0 for primary and 1 and above for secondary.
+        :param combine_screens: If True, a collage of all display screens will be returned. Otherwise, a list of display screens will be returned.
         """
         if not self.computer.emit_images and force_image == False:
             text = self.get_text_as_list_of_lists()
@@ -91,10 +106,7 @@ def screenshot(
                 region = self.get_active_window()["region"]
                 screenshot = pyautogui.screenshot(region=region)
             else:
-                if platform.system() == "Darwin":
-                    screenshot = take_screenshot_to_pil()
-                else:
-                    screenshot = pyautogui.screenshot()
+                screenshot = take_screenshot_to_pil(screen=screen, combine_screens=combine_screens) #  this function uses pyautogui.screenshot which works fine for all OS (mac, linux and windows)
                 # message = format_to_recipient("Taking a screenshot of the entire screen. This is not recommended. You (the language model assistant) will recieve it with low resolution.\n\nTo maximize performance, use computer.display.view(active_app_only=True). This will produce an ultra high quality image of the active application.", "assistant")
                 # print(message)
 
@@ -121,18 +133,26 @@ def screenshot(
 
         # Open the image file with PIL
         # IPython interactive mode auto-displays plots, causing RGBA handling issues, possibly MacOS-specific.
-        screenshot = screenshot.convert("RGB")
+        if isinstance(screenshot, list):
+            screenshot = [img.convert("RGB") for img in screenshot] # if screenshot is a list (i.e combine_screens=False).
+        else:
+            screenshot = screenshot.convert("RGB")
 
         if show:
             # Show the image using matplotlib
-            plt.imshow(np.array(screenshot))
+            if isinstance(screenshot, list):
+                for img in screenshot:
+                    plt.imshow(np.array(img))
+                    plt.show()
+            else:
+                plt.imshow(np.array(screenshot))
 
             with warnings.catch_warnings():
                 # It displays an annoying message about Agg not being able to display something or WHATEVER
                 warnings.simplefilter("ignore")
                 plt.show()
 
-        return screenshot
+        return screenshot # this will be a list of combine_screens == False
 
     def find(self, description, screenshot=None):
         if description.startswith('"') and description.endswith('"'):
@@ -260,22 +280,78 @@ def get_text_as_list_of_lists(self, screenshot=None):
             )
 
 
-import io
-import subprocess
-
-from PIL import Image
-
-
-def take_screenshot_to_pil(filename="temp_screenshot.png"):
-    # Capture the screenshot and save it to a temporary file
-    subprocess.run(["screencapture", "-x", filename], check=True)
-
-    # Open the image file with PIL
-    with open(filename, "rb") as f:
-        image_data = f.read()
-    image = Image.open(io.BytesIO(image_data))
-
-    # Optionally, delete the temporary file if you don't need it after loading
-    os.remove(filename)
-
-    return image
+def take_screenshot_to_pil(screen=0, combine_screens=True):
+    # Get information about all screens
+    monitors = get_monitors()
+    if screen == -1: # All screens
+        
+        # Take a screenshot of each screen and save them in a list
+        screenshots = [pyautogui.screenshot(region=(monitor.x, monitor.y, monitor.width, monitor.height)) for monitor in monitors]
+
+        if combine_screens:
+            # Combine all screenshots horizontally
+            total_width = sum([img.width for img in screenshots])
+            max_height = max([img.height for img in screenshots])
+
+            # Create a new image with a size that can contain all screenshots
+            new_img = Image.new('RGB', (total_width, max_height))
+
+            # Paste each screenshot into the new image
+            x_offset = 0
+            for i, img in enumerate(screenshots):
+                # Convert PIL Image to OpenCV Image (numpy array)
+                img_cv = np.array(img)
+                img_cv = cv2.cvtColor(img_cv, cv2.COLOR_RGB2BGR)
+
+                # Convert new_img PIL Image to OpenCV Image (numpy array)
+                new_img_cv = np.array(new_img)
+                new_img_cv = cv2.cvtColor(new_img_cv, cv2.COLOR_RGB2BGR)
+
+                # Paste each screenshot into the new image using OpenCV
+                new_img_cv[0:img_cv.shape[0], x_offset:x_offset+img_cv.shape[1]] = img_cv
+                x_offset += img.width
+
+                # Add monitor labels using OpenCV
+                font = cv2.FONT_HERSHEY_SIMPLEX
+                font_scale = 4
+                font_color = (255, 255, 255)
+                line_type = 2
+
+                if i == 0:
+                    text = "Primary Monitor"
+                else:
+                    text = f"Monitor {i}"
+                
+                # Calculate the font scale that will fit the text perfectly in the center of the monitor
+                text_size = cv2.getTextSize(text, font, font_scale, line_type)[0]
+                font_scale = min(img.width / text_size[0], img.height / text_size[1])
+                
+                # Recalculate the text size with the new font scale
+                text_size = cv2.getTextSize(text, font, font_scale, line_type)[0]
+                
+                # Calculate the position to center the text
+                text_x = x_offset - img.width // 2 - text_size[0] // 2
+                text_y = max_height // 2 - text_size[1] // 2
+                
+                cv2.putText(new_img_cv, text, (text_x, text_y), font, font_scale, font_color, line_type)
+
+                # Convert new_img from OpenCV Image back to PIL Image
+                new_img_cv = cv2.cvtColor(new_img_cv, cv2.COLOR_BGR2RGB)
+                new_img = Image.fromarray(new_img_cv)
+
+            return new_img
+        else:
+            return screenshots
+    elif screen > 0:
+        # Take a screenshot of the selected screen
+        return pyautogui.screenshot(region=(monitors[screen].x, monitors[screen].y, monitors[screen].width, monitors[screen].height))
+        
+    else:
+        # Take a screenshot of the primary screen
+        return pyautogui.screenshot(region=(monitors[screen].x, monitors[screen].y, monitors[screen].width, monitors[screen].height))
+
+
+def get_displays():
+    monitors = get_monitors()
+    return monitors
+    

interpreter/core/computer/terminal/languages/jupyter_language.py

@@ -26,16 +26,6 @@ class JupyterLanguage(BaseLanguage):
 
     def __init__(self, computer):
         self.computer = computer
-        # Filter out the following messages from IPKernelApp, to prevent the logs from showing up anytime someone presses CTRL+C
-        if not DEBUG_MODE:
-            ipkernel_logger = logging.getLogger('IPKernelApp')
-            # Create a filter using a lambda function
-            warning_filter = lambda record: not any(msg in record.getMessage() for msg in [
-                "Parent appears to have exited, shutting down.",
-                "Could not destroy zmq context"
-            ])
-            # Add the filter to the logger
-            ipkernel_logger.addFilter(warning_filter)
 
         self.km = KernelManager(kernel_name="python3")
         self.km.start_kernel()

interpreter/core/computer/terminal/terminal.py

@@ -37,12 +37,8 @@ def get_language(self, language):
         return None
 
     def run(self, language, code, stream=False, display=False):
-        if (
-            language == "python"
-            and self.computer.import_computer_api
-            and "computer" in code
-        ):
-            if not self.computer._has_imported_computer_api:
+        if language == "python":
+            if self.computer.import_computer_api and not self.computer._has_imported_computer_api and "computer" in code:
                 self.computer._has_imported_computer_api = True
                 # Give it access to the computer via Python
                 self.computer.run(
@@ -51,6 +47,10 @@ def run(self, language, code, stream=False, display=False):
                     display=self.computer.verbose,
                 )
 
+            if self.computer.import_skills and not self.computer._has_imported_skills:
+                self.computer._has_imported_skills = True
+                self.computer.skills.import_skills()
+
         if stream == False:
             # If stream == False, *pull* from _streaming_run.
             output_messages = []