Fixed a bug in text plus image content generation

Author: yeshaokai

amadeusgpt/agentic/video_llm.py

@@ -136,8 +136,8 @@ def speak(self, video_data):
         multi_image_content = self.prepare_multi_image_content(images)       
         self.update_history("system", self.system_prompt)
         self.update_history(
-            "user", "This video is about a mouse in its home cage.", multi_image_content=multi_image_content, in_place = True)
-         
+            "user", "This video is about a mouse in its home cage. The circular object in the center is a treadmill. The colorful dots are keypoints from DeepLabCut. You can ignore those dots.", multi_image_content=multi_image_content, in_place = True)
+
         response = self.connect_gpt(self.context_window)
         text = response.choices[0].message.content.strip()
         print(text)

amadeusgpt/analysis_objects/llm.py

@@ -154,6 +154,8 @@ def update_history(self, role, content, multi_image_content = None, in_place=Fal
             if multi_image_content is None:                              
                 new_message = {"role": role, "content": content}
             else:
+                text_content = {"type": "text", "text": content}
+                multi_image_content = [text_content] + multi_image_content
                 new_message = {"role": role, "content": multi_image_content
                   }