wip

Author: yeshaokai

amadeusgpt/agentic/video_llm.py

@@ -131,12 +131,14 @@ def speak(self, video_data):
             images.extend(segment)
         print ('total images GPT-4 sees', len(images))
         self.images_to_video(images, 'gpt4_sees_this.mp4', 30)
+     
+
         multi_image_content = self.prepare_multi_image_content(images)       
         self.update_history("system", self.system_prompt)
         self.update_history(
             "user", "This video is about a mouse in its home cage.", multi_image_content=multi_image_content, in_place = True)
-        
-        response = self.connect_gpt(self.context_window, max_tokens=2000)
+
+        response = self.connect_gpt(self.context_window)
         text = response.choices[0].message.content.strip()
         print(text)
 
@@ -151,5 +153,7 @@ def speak(self, video_data):
     video_sampler = VideoSampler(video_path, segment_duration, frames_per_segment)
     video_data = video_sampler.process_video()
 
+    video_data = {k:v for k,v in video_data.items() if k < 1}
+    print (video_data)
     video_llm = VideoLLM(config=config)
     video_llm.speak(video_data)

amadeusgpt/analysis_objects/llm.py

@@ -20,7 +20,7 @@ class LLM(AnalysisObject):
 
     def __init__(self, config):
         self.config = config
-        self.max_tokens = config.get("max_tokens", 6000)
+        self.max_tokens = config.get("max_tokens", 36000)
         self.gpt_model = config.get("gpt_model", "gpt-4o")
         self.keep_last_n_messages = config.get("keep_last_n_messages", 2)
 
@@ -366,11 +366,6 @@ def speak(self, sandbox):
 
 if __name__ == "__main__":
     from amadeusgpt.config import Config
-    from amadeusgpt.main import create_amadeus
 
-    config = Config("amadeusgpt/configs/EPM_template.yaml")
+    config = Config("/Users/shaokaiye/AmadeusGPT-dev/amadeusgpt/configs/MausHaus_template.yaml")
 
-    amadeus = create_amadeus(config)
-    sandbox = amadeus.sandbox
-    visualLLm = VisualLLM(config)
-    visualLLm.speak(sandbox)

amadeusgpt/main.py

@@ -23,7 +23,7 @@
 from amadeusgpt.programs.task_program_registry import TaskProgramLibrary
 
 class AMADEUS:
-    def __init__(self, config: Dict[str, Any]):
+    def __init__(self, config: Dict[str, Any] | Config):
         self.config = config
         self.code_generator_llm = CodeGenerationLLM(config.get("llm_info", {}))
         self.self_debug_llm = SelfDebugLLM(config.get("llm_info", {}))
@@ -52,7 +52,7 @@ def __init__(self, config: Dict[str, Any]):
             self.sandbox.register_llm("diagnosis", self.diagnosis_llm)
 
         # can only do this after the register process
-        self.sandbox.configure_using_vlm()
+        # self.sandbox.configure_using_vlm()
 
     def match_integration_module(self, user_query: str):
         """

amadeusgpt/programs/sandbox.py

@@ -312,6 +312,14 @@ def get_query_block(self):
     def update_config(self, config):
         self.config = config
         self.update_namespace()
+    
+    def get_analysis(self):
+        """
+        Every sandbox stores a unique "behavior analysis" instance in its namespace
+        Therefore, get analysis gets the current sandbox's analysis.
+        """
+        analysis = self.exec_namespace["behavior_analysis"]
+        return analysis
 
     def copy(self):
         return Sandbox(self.config, self.api_registry)