update update_history

Author: yeshaokai

amadeusgpt/analysis_objects/llm.py

@@ -118,7 +118,7 @@ def connect_gpt_oai_1(self, messages, **kwargs):
 
         return response
 
-    def update_history(self, role, content, encoded_image=None, replace=False):
+    def update_history(self, role, content, encoded_image = None, replace=False):
         if role == "system":
             if len(self.history) > 0:
                 self.history[0]["content"] = content
@@ -132,35 +132,26 @@ def update_history(self, role, content, encoded_image=None, replace=False):
                 self.history.append({"role": role, "content": content})
                 num_AI_messages = (len(self.context_window) - 1) // 2
                 if num_AI_messages == self.keep_last_n_messages:
-                    print("doing active forgetting")
+                    print ("doing active forgetting")
                     # we forget the oldest AI message and corresponding answer
                     self.context_window.pop(1)
                     self.context_window.pop(1)
                 new_message = {"role": role, "content": content}
             else:
-                if encoded_image is None:
-                    self.history.append({"role": role, "content": content})
-                    num_AI_messages = (len(self.context_window) - 1) // 2
-                    if num_AI_messages == self.keep_last_n_messages:
-                        print("doing active forgetting")
-                        # we forget the oldest AI message and corresponding answer
-                        self.context_window.pop(1)
-                        self.context_window.pop(1)
-                    self.context_window.append({"role": role, "content": content})
-                else:
-                    message = {
-                        "role": "user",
-                        "content": [
-                            {"type": "text", "text": content},
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": f"data:image/png;base64,{encoded_image}"
-                                },
-                            },
-                        ],
+                new_message = {"role": "user", "content": [
+                    {"type": "text", "text": ""},
+                    {"type": "image_url", "image_url": {
+                        "url": f"data:image/jpeg;base64,{encoded_image}"}
                     }
-                    self.context_window.append(message)
+                ]}
+
+            self.history.append(new_message)
+            
+            if replace == True:
+                if len(self.context_window) == 2:
+                    self.context_window[1] = new_message
+                else:
+                    self.context_window.append(new_message)
 
     def clean_context_window(self):
         while len(self.context_window) > 1:
@@ -288,66 +279,6 @@ def update_system_prompt(self, sandbox):
         # update both history and context window
         self.update_history("system", self.system_prompt)
 
-
-class MutationLLM(LLM):
-    def __init__(self, config):
-        super().__init__(config)
-
-    def update_system_prompt(self, sandbox):
-        from amadeusgpt.system_prompts.mutation import _get_system_prompt
-
-        self.system_prompt = _get_system_prompt(sandbox)
-        # update both history and context window
-        self.update_history("system", self.system_prompt)
-
-    def speak(self, sandbox):
-        # TODO maybe we don't need to keep the history
-        """
-        Speak to the chat channel
-        """
-        # query = "Please start. Make sure you provide one task program a time. Thanks a million!"
-        query = "Please start. Thanks a million!"
-        self.update_system_prompt(sandbox)
-        self.update_history("user", query, replace=True)
-        response = self.connect_gpt(self.context_window, max_tokens=4000)
-        text = response.choices[0].message.content.strip()
-        sandbox.chat_channel.chain_of_thought.append(response)
-        return text
-
-
-class BreedLLM(LLM):
-    def __init__(self, config):
-        super().__init__(config)
-
-    def update_system_prompt(self, sandbox):
-        from amadeusgpt.system_prompts.breed import _get_system_prompt
-
-        behavior1_docs, behavior2_docs, composition_type = sandbox.get_breed_info()
-
-        self.system_prompt = _get_system_prompt(
-            behavior1_docs, behavior2_docs, composition_type
-        )
-
-        # update both history and context window
-
-        self.update_history("system", self.system_prompt)
-
-    def speak(self, sandbox):
-        # TODO maybe we don't need to keep the history
-        """
-        Speak to the chat channel
-        """
-        query = "Now write the template function. Make sure your answer is concise and don't mention anything about filtering such as smooth_window or min_window\n"
-        self.update_system_prompt(sandbox)
-        self.update_history("user", query, replace=True)
-
-        response = self.connect_gpt(self.context_window, max_tokens=400)
-        text = response.choices[0].message.content.strip()
-        sandbox.chat_channel.chain_of_thought.append(response)
-
-        return text
-
-
 class DiagnosisLLM(LLM):
     """
     Resource management for testing and error handling

amadeusgpt/main.py

@@ -8,7 +8,6 @@
 from amadeusgpt.utils import *
 
 warnings.filterwarnings("ignore")
-import os
 
 from amadeusgpt.analysis_objects.llm import (CodeGenerationLLM, DiagnosisLLM,
                                              SelfDebugLLM, VisualLLM)
@@ -73,11 +72,7 @@ def match_integration_module(self, user_query: str):
                 modules.append(query_module)
 
                 # parse the query result by loading active loading
-        return modules
-
-    def chat_iteration(self, user_query):
-        qa_message = self.sandbox.llm_step(user_query)
-        return qa_message
+        return modules  
 
     def step(self, user_query):
         integration_module_names = self.match_integration_module(user_query)

amadeusgpt/programs/sandbox.py

@@ -399,7 +399,6 @@ def events_to_videos(self, events, function_name):
     def render_qa_message(self, qa_message):
         function_rets = qa_message["function_rets"]
         behavior_analysis = self.exec_namespace["behavior_analysis"]
-        n_animals = behavior_analysis.animal_manager.get_n_individuals()
         bodypart_names = behavior_analysis.animal_manager.get_keypoint_names()
         qa_message["pose_video"] = (
             behavior_analysis.animal_manager.superanimal_predicted_video
@@ -464,7 +463,6 @@ def llm_step(self, user_query):
             qa_message["meta_info"] = self.meta_info
 
         self.messages.append(qa_message)
-        post_process_llm = []  # ['self_debug', 'diagnosis']
         self.query = user_query
         self.llms["code_generator"].speak(self)
 
@@ -484,7 +482,14 @@ def run_task_program(self, task_program_name):
         return qa_message
 
     def step(self, user_query, number_of_debugs=1):
+        """
+        Currently not used. We tried to seperate LLM inference and code execution
+        """            
         qa_message = create_message(user_query, self)
+
+        if self.meta_info is not None:
+            qa_message["meta_info"] = self.meta_info
+
         self.messages.append(qa_message)
 
         self.query = user_query
@@ -595,68 +600,4 @@ def render_temp_message(query, sandbox):
     for name, roi_object in roi_objects.items():
         analysis.object_manager.add_roi_object(ROIObject(name, roi_object["Path"]))
 
-    render_temp_message("random query", sandbox)
-
-    # def get_head_dips_events(config: Config):
-    #     """
-    #     Identify and count the number of head_dips events.
-
-    #     Parameters:
-    #     ----------
-    #     config: Config
-
-    #     Returns:
-    #     -------
-    #     head_dips_events: List[BaseEvent]
-    #         List of events where head_dips behavior occurs.
-    #     num_bouts: int
-    #         Number of bouts for head_dips behavior.
-    #     """
-    #     # Create an instance of AnimalBehaviorAnalysis
-    #     analysis = create_analysis(config)
-
-    #     # Get events where mouse_center and neck are inside ROI0
-    #     mouse_center_neck_in_ROI0_events = analysis.get_animals_object_events(
-    #         object_name='ROI0',
-    #         query='overlap == True',
-    #         bodypart_names=['mouse_center', 'neck'],
-    #         min_window=1,
-    #         max_window=100000,
-    #         negate=False
-    #     )
-    #     # print ("mouse center neck in ROI0")
-    #     # print (len(mouse_center_neck_in_ROI0_events))
-    #     # for event in mouse_center_neck_in_ROI0_events:
-    #     #     print (event)
-
-    #     # Get events where head_midpoint is outside ROI1
-    #     head_midpoint_outside_ROI1_events = analysis.get_animals_object_events(
-    #         object_name='ROI1',
-    #         query='overlap == True',
-    #         bodypart_names=['head_midpoint'],
-    #         min_window=1,
-    #         max_window=100000,
-    #         negate=True
-    #     )
-    #     # print ('mouse head not in ROI1')
-    #     # print (len(head_midpoint_outside_ROI1_events))
-    #     # for event in head_midpoint_outside_ROI1_events:
-    #     #     print (event)
-
-    #     # Combine the events to define head_dips behavior
-    #     head_dips_events = analysis.get_composite_events(
-    #         events_A=mouse_center_neck_in_ROI0_events,
-    #         events_B=head_midpoint_outside_ROI1_events,
-    #         composition_type='logical_and',
-    #         max_interval_between_sequential_events=0,
-    #         min_window=1,
-    #         max_window=100000
-    #     )
-    #     print ('head dips events', len(head_dips_events))
-
-    #     # Count the number of bouts for head_dips behavior
-    #     num_bouts = len(head_dips_events)
-
-    #     return head_dips_events, num_bouts
-
-    # get_head_dips_events(config)
+    render_temp_message("random query", sandbox)