merge performance changes from dev branch including NaN handling update and frame crash handling and streaming fix

Author: JensBlack

DeepLabStream.py

@@ -281,38 +281,42 @@ def get_pose_mp(input_q, output_q):
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     if MODEL_ORIGIN == 'DLC':
                         scmap, locref, pose = get_pose(frame, config, sess, inputs, outputs)
                         # TODO: Remove alterations to original
                         peaks = find_local_peaks_new(scmap, locref, ANIMALS_NUMBER, config)
                         # peaks = pose
                     if MODEL_ORIGIN == 'MADLC':
                         peaks = get_ma_pose(frame, config, sess, inputs, outputs)
-
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index, peaks, analysis_time))
 
         elif MODEL_ORIGIN == 'DLC-LIVE':
             dlc_live = load_dlc_live()
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     if not dlc_live.is_initialized:
                         peaks = dlc_live.init_inference(frame)
                     else:
                         peaks = dlc_live.get_pose(frame)
-
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index, peaks, analysis_time))
 
         elif MODEL_ORIGIN == 'DEEPPOSEKIT':
             predict_model = load_dpk()
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     frame = frame[..., 1][..., None]
                     st_frame = np.stack([frame])
                     prediction = predict_model.predict(st_frame, batch_size=1, verbose=True)
                     peaks = prediction[0, :, :2]
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index,peaks,analysis_time))
         else:
             raise ValueError(f'Model origin {MODEL_ORIGIN} not available.')
 
@@ -383,9 +387,9 @@ def input_frames_for_analysis(self, frames: tuple, index: int):
                     frame_time = time.time()
                     self._multiprocessing[camera]['input'].put((index, frame))
                     if d_maps:
-                        self.store_frames(camera, frame, d_maps[camera], frame_time)
+                        self.store_frames(camera, frame, d_maps[camera], frame_time, index)
                     else:
-                        self.store_frames(camera, frame, None, frame_time)
+                        self.store_frames(camera, frame, None, frame_time, index)
 
     def get_analysed_frames(self) -> tuple:
         """
@@ -405,12 +409,11 @@ def get_analysed_frames(self) -> tuple:
                         self._start_time = time.time()  # getting the first frame here
 
                     # Getting the analysed data
-                    analysed_index, peaks = self._multiprocessing[camera]['output'].get()
+                    analysed_index, peaks, analysis_time = self._multiprocessing[camera]['output'].get()
                     skeletons = calculate_skeletons(peaks, ANIMALS_NUMBER)
                     print('', end='\r', flush=True)  # this is the line you should not remove
-                    analysed_frame, depth_map, input_time = self.get_stored_frames(camera)
-                    analysis_time = time.time() - input_time
-
+                    analysed_frame , depth_map, input_time = self.get_stored_frames(camera, analysed_index)
+                    delay_time = time.time() - input_time
                     # Calculating FPS and plotting the data on frame
                     self.calculate_fps(analysis_time if analysis_time != 0 else 0.01)
                     frame_time = time.time() - self._start_time
@@ -434,23 +437,30 @@ def get_analysed_frames(self) -> tuple:
                     analysed_frames[camera] = analysed_image
             return analysed_frames, analysis_time
 
-    def store_frames(self, camera: str, c_frame, d_map, frame_time: float):
+    def store_frames(self, camera: str, c_frame, d_map, frame_time: float, index: int):
         """
-        Store frames currently sent for analysis
+        Store frames currently sent for analysis in index based dictionary
         :param camera: camera name
         :param c_frame: color frame
         :param d_map: depth map
         :param frame_time: inputting time of frameset
+        :param index: index of frame that is currently analysed
         """
-        self._stored_frames[camera] = c_frame, d_map, frame_time
+        if camera in self._stored_frames.keys():
+            self._stored_frames[camera][index] = c_frame, d_map, frame_time
+
+        else:
+            self._stored_frames[camera] = {}
+            self._stored_frames[camera][index] = c_frame, d_map, frame_time
 
-    def get_stored_frames(self, camera: str):
+    def get_stored_frames(self, camera: str, index: int):
         """
-        Retrieve frames currently sent for analysis
+        Retrieve frames currently sent for analysis, retrieved frames will be removed (popped) from the dictionary
         :param camera: camera name
+        :param index: index of analysed frame
         :return:
         """
-        c_frame, d_map, frame_time = self._stored_frames.get(camera)
+        c_frame, d_map, frame_time = self._stored_frames[camera].pop(index, None)
         return c_frame, d_map, frame_time
 
     def convert_depth_map_to_image(self, d_map):

app.py

@@ -15,10 +15,9 @@
 from utils.configloader import MULTI_CAM, STREAMS, RECORD_EXP
 from utils.gui_image import QFrame, ImageWindow, emit_qframes
 
-from PyQt5.QtCore import QThread
-from PyQt5.QtWidgets import QPushButton, QApplication, QWidget, QGridLayout
-from PyQt5.QtGui import QIcon
-
+from PySide2.QtCore import QThread
+from PySide2.QtWidgets import QPushButton, QApplication, QWidget, QGridLayout
+from PySide2.QtGui import QIcon
 
 # creating a complete thread process to work in the background
 class AThread(QThread):

utils/poser.py

@@ -335,13 +335,14 @@ def handle_missing_bp(animal_skeletons: list):
     :return animal_skeleton with handled missing values"""
     for skeleton in animal_skeletons:
         for bodypart, coordinates in skeleton.items():
-            if any(np.isnan(coordinates)):
+            np_coords = np.array((coordinates))
+            if any(np.isnan(np_coords)):
                 if HANDLE_MISSING == 'skip':
                     animal_skeletons.remove(skeleton)
                     break
                 elif HANDLE_MISSING == 'null':
-                    new_coordinates = np.nan_to_num(coordinates, copy = True)
-                    skeleton[bodypart] = new_coordinates
+                    new_coordinates = np.nan_to_num(np_coords, copy = True)
+                    skeleton[bodypart] = tuple(new_coordinates)
                 else:
                     animal_skeletons.remove(skeleton)
                     break