temporally removed docker

Author: yeshaokai

amadeusgpt/analysis_objects/llm.py

@@ -3,10 +3,11 @@
 import re
 import time
 import traceback
-from amadeusgpt.utils import AmadeusLogger, search_generated_func
+from amadeusgpt.utils import AmadeusLogger
 from .base import AnalysisObject
 import openai
 from openai import OpenAI
+import base64
 
 class LLM(AnalysisObject):
     total_tokens = 0
@@ -23,13 +24,11 @@ def __init__(self, config):
         self.context_window = []
         # only for logging and long-term memory usage.
         self.history = []
+    
 
-    def whetehr_speak(self):
-        """
-        Handcrafted rules to decide whether to speak
-        1) If there is a error in the current chat channel
-        """
-        return False
+    def encode_image(self, image_path):
+        with open(image_path, "rb") as image_file:
+            return base64.b64encode(image_file.read()).decode("utf-8")
 
     def speak(self):
         """
@@ -41,7 +40,10 @@ def connect_gpt(self, messages, **kwargs):
         # if openai version is less than 1
         return self.connect_gpt_oai_1(messages, **kwargs)
 
-    def connect_gpt_oai_1(self, messages, **kwargs):        
+    def connect_gpt_oai_1(self, messages, **kwargs): 
+        """
+        This is routed to openai > 1.0 interfaces
+        """       
 
         if self.config.get("use_streamlit", False):
             if "OPENAI_API_KEY" in os.environ:
@@ -68,10 +70,6 @@ def connect_gpt_oai_1(self, messages, **kwargs):
         # the usage was recorded from the last run. However, since we have many LLMs that
         # share the call of this function, we will need to store usage and retrieve them from the database class
         num_retries = 3
-        print ('number of messages to send', len(messages))
-        print ('print the message')
-        for message in messages:
-            print (message)
         for _ in range(num_retries):
             try:
                 json_data = {
@@ -113,7 +111,7 @@ def connect_gpt_oai_1(self, messages, **kwargs):
 
         return response
 
-    def update_history(self, role, content, replace=False):
+    def update_history(self, role, content, encoded_image = None, replace=False):
         if role == "system":
             if len(self.history) > 0:
                 self.history[0]["content"] = content
@@ -132,17 +130,27 @@ def update_history(self, role, content, replace=False):
                     self.context_window.append({"role": role, "content": content})
 
             else:
-
-                self.history.append({"role": role, "content": content})
-
-                num_AI_messages = (len(self.context_window) - 1) // 2
-                if num_AI_messages == self.keep_last_n_messages:
-                    print ("doing active forgetting")
-                    # we forget the oldest AI message and corresponding answer
-                    self.context_window.pop(1)
-                    self.context_window.pop(1)
-
-                self.context_window.append({"role": role, "content": content})
+                if encoded_image is None:
+                    self.history.append({"role": role, "content": content})
+                    num_AI_messages = (len(self.context_window) - 1) // 2
+                    if num_AI_messages == self.keep_last_n_messages:
+                        print ("doing active forgetting")
+                        # we forget the oldest AI message and corresponding answer
+                        self.context_window.pop(1)
+                        self.context_window.pop(1)
+                    self.context_window.append({"role": role, "content": content})
+                else:
+                    message = {
+                        "role": "user", "content": [
+                        {"type": "text", "text": content},
+                        {"type": "image_url", "image_url": {
+                        "url": f"data:image/png;base64,{encoded_image}"}
+                        }]
+                    }                                            
+                    self.context_window.append(message) 
+    
+                    
+               
 
     def clean_context_window(self):
         while len(self.context_window) > 1:
@@ -174,19 +182,34 @@ def parse_openai_response(cls, response):
         return text, function_code, thought_process
 
 
+class VisualLLM(LLM):
+    def __init__(self, config):
+        super().__init__(config)
+    def speak(self, sandbox):
+        """
+        Only to comment about one image
+        #1) What animal is there, how many and what superanimal model we should use
+        #2) report the background object list
+        #3) We format them in json format
+
+        """
+
+        from amadeusgpt.system_prompts.visual import _get_system_prompt
+        self.system_prompt = _get_system_prompt()
+        analysis = sandbox.exec_namespace["behavior_analysis"]
+        scene_image = analysis.visual_manager.get_scene_image()
+        encoded_image = self.encode_image(scene_image)
+        self.update_history("user", encoded_image)
+
+
 class CodeGenerationLLM(LLM):
     """
     Resource management for the behavior analysis part of the system
     """
 
     def __init__(self, config):
         super().__init__(config)
-
-    def whether_speak(self, sandbox):
-        """
-        1) if there is a error from last iteration, don't speak
-        """
-        return True
+  
 
     def speak(self, sandbox):
         """
@@ -244,17 +267,7 @@ def update_system_prompt(self, sandbox):
 class MutationLLM(LLM):
     def __init__(self, config):
         super().__init__(config)
-
-    def whether_speak(self, chat_channel):
-        """
-        1) if there is a error from last iteration, don't speak
-        """
-
-        error = chat_channel.get_last_message().get("error", None)
-        if error is not None:
-            return False
-        else:
-            return True
+    
 
     def update_system_prompt(self, sandbox):
         from amadeusgpt.system_prompts.mutation import _get_system_prompt
@@ -281,17 +294,7 @@ def speak(self, sandbox):
 class BreedLLM(LLM):
     def __init__(self, config):
         super().__init__(config)
-
-    def whether_speak(self, chat_channel):
-        """
-        1) if there is a error from last iteration, don't speak
-        """
-
-        error = chat_channel.get_last_message().get("error", None)
-        if error is not None:
-            return False
-        else:
-            return True
+    
 
     def update_system_prompt(self, sandbox):
         from amadeusgpt.system_prompts.breed import _get_system_prompt
@@ -326,18 +329,7 @@ class DiagnosisLLM(LLM):
     """
     Resource management for testing and error handling
     """
-
-    def whether_speak(self, chat_channel):
-        """
-        Handcrafted rules to decide whether to speak
-        1) If there is a error in the current chat channel
-        """
-        if chat_channel.get_last_message() is None:
-            return False
-        else:
-            error = chat_channel.get_last_message().get("error", None)
-
-            return error is None
+    
 
     @classmethod
     def get_system_prompt(

conda/amadesuGPT-cpu.yml

@@ -3,7 +3,7 @@ name: amadeusgpt-cpu
 channels:
     - defaults
 dependencies:
-    - python<3.10
+    - python==3.10
     - pytables==3.8.0
     - hdf5
     - pip

conda/amadesuGPT-minimal.yml

@@ -3,7 +3,7 @@ name: amadeusgpt-minimal
 channels:
     - defaults
 dependencies:
-    - python<3.10
+    - python==3.10
     - hdf5
     - pip
     - jupyter

docker/Dockerfile

@@ -1,4 +1,5 @@
 #!/bin/bash
+# change this to your own miniconda / miniforge path
 source /Users/shaokaiye/miniforge3/bin/activate
 conda env create -f conda/amadesuGPT-minimal.yml
 conda activate amadeusgpt-minimal

docker/Makefile

@@ -7,7 +7,7 @@ name = "AmadeusGPT"
 readme = "README.md"
 requires-python = ">=3.8"
 dynamic = ["version"]
-version = "0.0.2"
+version = "0.0.3"
 
 [tool.setuptools]
 packages = ["amadeusgpt"]