Llama 3 Vision

Author: KillianLucas

interpreter/core/computer/computer.py

@@ -17,6 +17,7 @@
 from .terminal.terminal import Terminal
 from .vision.vision import Vision
 
+
 class Computer:
     def __init__(self, interpreter):
         self.interpreter = interpreter
@@ -93,6 +94,12 @@ def screenshot(self, *args, **kwargs):
         """
         return self.display.screenshot(*args, **kwargs)
 
+    def view(self, *args, **kwargs):
+        """
+        Shortcut for computer.display.screenshot
+        """
+        return self.display.screenshot(*args, **kwargs)
+
     def to_dict(self):
         def json_serializable(obj):
             try:

interpreter/core/computer/display/display.py

@@ -94,7 +94,13 @@ def screenshot(
         :param combine_screens: If True, a collage of all display screens will be returned. Otherwise, a list of display screens will be returned.
         """
         if not self.computer.emit_images and force_image == False:
-            text = self.get_text_as_list_of_lists()
+            screenshot = self.screenshot(show=False, force_image=True)
+
+            description = self.computer.vision.query(pil_image=screenshot)
+            print("A DESCRIPTION OF WHAT'S ON THE SCREEN: " + description)
+
+            print("ALL OF THE TEXT ON THE SCREEN: ")
+            text = self.get_text_as_list_of_lists(screenshot=screenshot)
             pp = pprint.PrettyPrinter(indent=4)
             pretty_text = pp.pformat(text)  # language models like it pretty!
             pretty_text = format_to_recipient(pretty_text, "assistant")

interpreter/core/computer/vision/vision.py

@@ -5,7 +5,7 @@
 
 from ...utils.lazy_import import lazy_import
 
-transformers = lazy_import("transformers")
+# transformers = lazy_import("transformers") # Doesn't work for some reason! We import it later.
 
 
 class Vision:
@@ -15,6 +15,8 @@ def __init__(self, computer):
         self.tokenizer = None  # Will load upon first use
 
     def load(self):
+        import transformers  # Wait until we use it. Transformers can't be lazy loaded for some reason!
+
         print(
             "Open Interpreter will use Moondream (tiny vision model) to describe images to the language model. Set `interpreter.llm.vision_renderer = None` to disable this behavior."
         )
@@ -30,7 +32,14 @@ def load(self):
             model_id, revision=revision
         )
 
-    def query(self, query="Describe this image.", base_64=None, path=None, lmc=None):
+    def query(
+        self,
+        query="Describe this image.",
+        base_64=None,
+        path=None,
+        lmc=None,
+        pil_image=None,
+    ):
         """
         Uses Moondream to ask query of the image (which can be a base64, path, or lmc message)
         """
@@ -59,6 +68,8 @@ def query(self, query="Describe this image.", base_64=None, path=None, lmc=None)
             img = Image.open(io.BytesIO(img_data))
         elif path:
             img = Image.open(path)
+        elif pil_image:
+            img = pil_image
 
         enc_image = self.model.encode_image(img)
         return self.model.answer_question(enc_image, query, self.tokenizer)

tests/test_interpreter.py

@@ -21,6 +21,14 @@
 import pytest
 from websocket import create_connection
 
+
+def test_localos():
+    interpreter.computer.emit_images = False
+    interpreter.computer.view()
+    interpreter.computer.emit_images = True
+    assert False
+
+
 @pytest.mark.skip(reason="Requires open-interpreter[local]")
 def test_m_vision():
     base64png = "iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADMElEQVR4nOzVwQnAIBQFQYXff81RUkQCOyDj1YOPnbXWPmeTRef+/3O/OyBjzh3CD95BfqICMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMK0CMO0TAAD//2Anhf4QtqobAAAAAElFTkSuQmCC"
@@ -45,8 +53,10 @@ def test_m_vision():
 
     interpreter.force_task_completion = False
     import time
+
     time.sleep(10)
 
+
 @pytest.mark.skip(reason="Computer with display only + no way to fail test")
 def test_point():
     # interpreter.computer.debug = True