Fixed some sort of rebase bug

Author: yeshaokai

amadeusgpt/analysis_objects/llm.py

@@ -198,17 +198,22 @@ def speak(self, sandbox):
         self.system_prompt = _get_system_prompt()
         analysis = sandbox.exec_namespace["behavior_analysis"]
         scene_image = analysis.visual_manager.get_scene_image()
-        encoded_image = self.encode_image(scene_image)
-        self.update_history("user", encoded_image)
+        result, buffer = cv2.imencode('.jpeg', scene_image)     
+        image_bytes = io.BytesIO(buffer)
+        base64_image = base64.b64encode(image_bytes.getvalue()).decode('utf-8')   
 
+        self.update_history("system", self.system_prompt)
+        self.update_history("user", "here is the image", encoded_image = base64_image, replace = True)
+        response = self.connect_gpt(self.context_window, max_tokens=2000)        
+        text = response.choices[0].message.content.strip()
         print (text)
         pattern = r"```json(.*?)```"
         if len(re.findall(pattern, text, re.DOTALL)) == 0:
             raise ValueError("can't parse the json string correctly", text)
         else:
             json_string = re.findall(pattern, text, re.DOTALL)[0]
             json_obj = json.loads(json_string)
-            return json_obj
+            return json_obj      
 
 class CodeGenerationLLM(LLM):
     """

amadeusgpt/analysis_objects/object.py

@@ -298,7 +298,7 @@ def set_body_orientation_keypoints(
 
     def set_head_orientation_keypoints(
         self, head_orientation_keypoints: Dict[str, Any]
-    ):
+    ):        
         self.nose_name = head_orientation_keypoints["nose"]
         self.neck_name = head_orientation_keypoints["neck"]
         self.support_head_orientation = True

amadeusgpt/analysis_objects/relationship.py

@@ -92,7 +92,6 @@ def get_name(self):
         return self.__name__
 
     def query_relationship(self, query_name: str) -> ndarray:
-
         ret = self.data[query_name]
         return ret
 
@@ -244,16 +243,25 @@ def _animal_animal_relationship(
                 head_cs_inv, receiver_animal.get_center()
             )
 
-        relative_velocity = (
-            sender_animal.get_velocity() - receiver_animal.get_velocity()
-        )
-        relative_velocity_magnitude = np.linalg.norm(relative_velocity, axis=2)
-        # Then, average these magnitudes over all keypoints for each frame
-        relative_speed = np.nanmean(relative_velocity_magnitude, axis=1)
+        # relative_velocity = (
+        #     sender_animal.get_velocity() - receiver_animal.get_velocity()
+        # )
+        # relative_velocity_magnitude = np.linalg.norm(relative_velocity, axis=2)
+        # # Then, average these magnitudes over all keypoints for each frame
+        # relative_speed = np.nanmean(relative_velocity_magnitude, axis=1)
+
+        sender_pos = sender_animal.get_center()
+        receiver_pos = receiver_animal.get_center()
+        direction_vector = receiver_pos - sender_pos
+        sender_velocity = np.nanmean(sender_animal.get_velocity(), axis = 1)
+        norm_direction_vector = direction_vector / np.linalg.norm(direction_vector)
+        relative_speed = np.einsum('ij,ij->i', sender_velocity, norm_direction_vector)
+
         closest_distance = np.nanmin(
             get_pairwise_distance(sender_animal.keypoints, receiver_animal.keypoints),
             axis=(1, 2),
         )
+        print ('relative_speed', relative_speed.mean())
         ret = {
             "distance": distance,
             "overlap": overlap,
@@ -267,6 +275,6 @@ def _animal_animal_relationship(
         if angles is not None:
             ret["relative_angle"] = angles
         if orientation is not None:
-            ret["orientation"] = orientation
+            ret["orientation"] = orientation 
 
         return ret

amadeusgpt/main.py

@@ -89,6 +89,9 @@ def get_analysis(self):
         sandbox = self.sandbox
         analysis = sandbox.exec_namespace['behavior_analysis']
         return analysis
+    
+    def run_task_program(self, task_program_name: str):
+        return self.sandbox.run_task_program(task_program_name)
 
 
 if __name__ == "__main__":

amadeusgpt/managers/animal_manager.py

@@ -123,7 +123,8 @@ def init_pose(self):
                 animalseq.set_body_orientation_keypoints(
                     self.config["keypoint_info"]["body_orientation_keypoints"]
                 )
-            if "head_orientation_keypoints" in self.config["keypoint_info"]:
+            
+            if "head_orientation_keypoints" in self.config["keypoint_info"]:             
                 animalseq.set_head_orientation_keypoints(
                     self.config["keypoint_info"]["head_orientation_keypoints"]
                 )

amadeusgpt/managers/event_manager.py

@@ -1,14 +1,10 @@
 import re
 from typing import Any, Dict, List, Literal, Optional, Set, Union
-
 import numpy as np
-
-from amadeusgpt.analysis_objects import event
 from amadeusgpt.analysis_objects.event import BaseEvent, Event, EventGraph
 from amadeusgpt.analysis_objects.relationship import Orientation, Relationship
 from amadeusgpt.programs.api_registry import (register_class_methods,
                                               register_core_api)
-from amadeusgpt.utils import timer_decorator
 
 from .animal_manager import AnimalManager
 from .base import Manager, cache_decorator
@@ -182,8 +178,9 @@ def get_animals_state_events(
             self.animal_manager.update_roi_keypoint_by_names(bodypart_names)
         ret_events = []
         pattern = r"(==|<=|>=|<|>)"
-        comparison_operator = re.findall(pattern, query)[0]
-        query_name = query.split(comparison_operator)[0]
+        # note we need to strip off the spaces
+        comparison_operator = re.findall(pattern, query)[0].strip()
+        query_name = query.split(comparison_operator)[0].strip()
         comparison = comparison_operator + "".join(query.split(comparison_operator)[1:])
 
         for sender_animal_name in self.animal_manager.get_animal_names():
@@ -233,7 +230,6 @@ def get_events_from_relationship(
 
         mask = relationship.query_relationship(relation_query)
 
-        #print ('mask', mask)
         # determine whether the mask is a numpy of float or numpy of boolean
 
         if mask.dtype != bool:
@@ -318,9 +314,10 @@ def get_animals_animals_events(
             for query in cross_animal_query_list:
                 # assert that query must contain one of the operators
                 # find the operator
+                # note we need to strip the spaces
+                comparison_operator = re.findall(pattern, query)[0].strip()
+                _query = query.split(comparison_operator)[0].strip()
 
-                comparison_operator = re.findall(pattern, query)[0]
-                _query = query.split(comparison_operator)[0]
                 _comparison = comparison_operator + "".join(
                     query.split(comparison_operator)[1:]
                 )

amadeusgpt/managers/visual_manager.py

@@ -1,15 +1,11 @@
 import glob
 import os
-from calendar import c
-from sched import Event
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 import cv2
 import matplotlib.pyplot as plt
 import numpy as np
-import streamlit as st
 from matplotlib.patches import Wedge
-
 from amadeusgpt.analysis_objects.event import BaseEvent
 from amadeusgpt.analysis_objects.object import AnimalSeq
 from amadeusgpt.analysis_objects.visualization import (EventVisualization,

amadeusgpt/programs/sandbox.py

@@ -6,16 +6,12 @@
 import traceback
 import typing
 from functools import wraps
-
 import matplotlib.pyplot as plt
 import numpy as np
-
 from amadeusgpt.analysis_objects.analysis_factory import create_analysis
 from amadeusgpt.analysis_objects.event import BaseEvent
 from amadeusgpt.analysis_objects.relationship import Orientation
 from amadeusgpt.config import Config
-from amadeusgpt.implementation import AnimalBehaviorAnalysis
-from amadeusgpt.managers import visual_manager
 from amadeusgpt.programs.api_registry import (CORE_API_REGISTRY,
                                               INTEGRATION_API_REGISTRY)
 from amadeusgpt.programs.task_program_registry import (TaskProgram,
@@ -462,12 +458,24 @@ def llm_step(self, user_query):
 
 
         return qa_message
+    
+    def run_task_program(self, task_program_name):
+        """
+        Sandbox is also responsible for running task program
+        """
+        task_program = self.task_program_library[task_program_name]
+        self.query = "run the task program"
+        qa_message = create_message(self.query, self)
+        qa_message["code"] = task_program["source_code"]
+        self.messages.append(qa_message)  
+        self.code_execution(qa_message)
+        qa_message = self.render_qa_message(qa_message)
+        return qa_message              
 
     def step(self, user_query, number_of_debugs=1):
         qa_message = create_message(user_query, self)
         self.messages.append(qa_message)
 
-        post_process_llm = ["self_debug"]
         self.query = user_query
         self.llms["code_generator"].speak(self)
         # all these llms collectively compose a amadeus_answer

amadeusgpt/programs/task_program_registry.py

@@ -1,23 +1,8 @@
 import ast
 import json
-import os
-import typing
 from collections import defaultdict
-from typing import Any, Callable, Dict, List
-
-from numpy import ndarray
-
-from amadeusgpt.config import Config
-from amadeusgpt.implementation import AnimalBehaviorAnalysis
+from typing import Any, Callable
 from amadeusgpt.utils import func2json
-
-required_classes = {"AnimalBehaviorAnalysis": AnimalBehaviorAnalysis, "Config": Config}
-required_types = {
-    name: getattr(typing, name) for name in dir(typing) if not name.startswith("_")
-}
-required_types.update({"ndarray": ndarray})
-
-
 class TaskProgram:
     """
     The task program in the system should be uniquely tracked by the id
@@ -46,9 +31,7 @@ def task_program_name(config) -> List[BaseEvent]:
     __call__(): should take the context and run the program in a sandbox.
     In the future we use docker container to run it
 
-    """
-
-    exec_namespace = None
+    """    
     cache = defaultdict(dict)
 
     def __init__(
@@ -76,10 +59,17 @@ def __init__(
         self.json_obj["parents"] = parents
         self.json_obj["mutation_from"] = mutation_from
         self.json_obj["generation"] = generation
+        
 
     def __setitem__(self, key, value):
         self.json_obj[key] = value
 
+    def display(self):
+        print (self.json_obj['name'],
+                self.json_obj['source_code'],
+                self.json_obj['description'])
+               
+
     def __getitem__(self, key):
         """
         {
@@ -130,40 +120,15 @@ def deserialize(self):
 
     def validate(self):
         pass
-
-    def __call__(self, config) -> Any:
-        namespace = TaskProgram.exec_namespace
-        function_name = self.json_obj["name"]
-        keypoint_file = config["keypoint_info"]["keypoint_file_path"]
-        keypoint_type = keypoint_file.split(".")[-1]
-        videoname = keypoint_file.split("/")[-1].replace(keypoint_type, "")
-        if self.json_obj["source_code"] is not None:
-            exec(self.json_obj["source_code"], namespace)
-            function = namespace[function_name]
-        else:
-            assert self.json_obj["func_pointer"] is not None
-            function = self.json_obj["func_pointer"]
-            namespace[function_name] = function
-        call_str = f"{function_name}(config)"
-        if videoname in TaskProgram.cache[function_name]:
-            return TaskProgram.cache[function_name][videoname]
-        else:
-            exec(f"result = {call_str}", namespace)
-            result = namespace["result"]
-
-        return result
-
+  
 
 class TaskProgramLibrary:
     """
     Keep track of the task programs
     There are following types of task programs:
     1) Custom task programs that are created by the user (can be loaded from disk)
     2) Task programs that are created by LLMs
-
-
     """
-
     LIBRARY = {}
 
     @classmethod
@@ -217,13 +182,6 @@ def get_task_programs(cls):
         """
         return cls.LIBRARY
 
-    @classmethod
-    def bind_exec_namespace(cls, exec_namespace):
-        """
-        For task programs to execute, we need to bind the namespace
-        """
-        TaskProgram.exec_namespace = exec_namespace
-
     @classmethod
     def save(cls, out_path):
         ret = []

notebooks/EPM_demo.ipynb

@@ -14,8 +14,7 @@
     "import amadeusgpt\n",
     "from pathlib import Path\n",
     "import matplotlib.pyplot as plt\n",
-    "import cv2\n",
-    "from amadeusgpt.managers.gui_manager import ROISelector"
+    "import cv2"
    ]
   },
   {