Merge branch 'master' into dev_classification

# Conflicts:
#	experiments/custom/experiments.py
#	requirements.txt
#	settings.ini
#	utils/poser.py

Author: JensBlack

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: monthly
+    time: "04:00"
+  open-pull-requests-limit: 10

DeepLabStream.py

@@ -17,13 +17,13 @@
 import numpy as np
 import pandas as pd
 
-from utils.generic import VideoManager, WebCamManager, GenericManager
-from utils.configloader import RESOLUTION, FRAMERATE, OUT_DIR, MODEL_NAME, MULTI_CAM, STACK_FRAMES, \
-    ANIMALS_NUMBER, STREAMS, STREAMING_SOURCE, MODEL_ORIGIN
-from utils.plotter import plot_bodyparts, plot_metadata_frame
-from utils.poser import load_deeplabcut, load_dpk, load_dlc_live, get_pose, calculate_skeletons,\
-    find_local_peaks_new, get_ma_pose
+from utils.generic import VideoManager, GenericManager
 
+from utils.configloader import RESOLUTION, FRAMERATE, OUT_DIR, MODEL_NAME, MULTI_CAM, STACK_FRAMES, \
+    ANIMALS_NUMBER, FLATTEN_MA, PASS_SEPARATE, STREAMS, STREAMING_SOURCE, MODEL_ORIGIN, CROP, CROP_X, CROP_Y
+from utils.plotter import plot_bodyparts,plot_metadata_frame
+from utils.poser import load_deeplabcut,load_dpk,load_dlc_live,load_sleap, get_pose,calculate_skeletons, \
+    find_local_peaks_new,get_ma_pose
 
 def create_video_files(directory, devices, resolution, framerate, codec):
     """
@@ -146,6 +146,7 @@ def select_camera_manager():
                 pylon_manager = PylonManager()
                 manager_list.append(pylon_manager)
 
+
             def check_for_cameras(camera_manager):
                 """
                 Helper method to get cameras, connected to that camera manager
@@ -168,17 +169,21 @@ def check_for_cameras(camera_manager):
 
         MANAGER_SOURCE = {
             'video': VideoManager,
-            'ipwebcam': WebCamManager,
             'camera': select_camera_manager
         }
 
+        # loading WebCam manager, if installed
+        if find_spec("pyzmq") is not None:
+            from utils.webcam import WebCamManager
+            MANAGER_SOURCE['ipwebcam'] = WebCamManager()
+
         # initialize selected manager
         camera_manager = MANAGER_SOURCE.get(STREAMING_SOURCE)()
         if camera_manager is not None:
             return camera_manager
         else:
             raise ValueError(f'Streaming source {STREAMING_SOURCE} is not a valid option. \n'
-                             f'Please choose from "video", "camera" or "ipwebcam".')
+                             f'Please choose from "video", "camera" or "ipwebcam". Make sure that if you are using "ipwebcam" you installed the additional dependencies.')
 
     @property
     def cameras(self):
@@ -281,38 +286,61 @@ def get_pose_mp(input_q, output_q):
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     if MODEL_ORIGIN == 'DLC':
                         scmap, locref, pose = get_pose(frame, config, sess, inputs, outputs)
-                        # TODO: Remove alterations to original
                         peaks = find_local_peaks_new(scmap, locref, ANIMALS_NUMBER, config)
+                        #Use the line below to use raw DLC output rather then DLStream optimization
                         # peaks = pose
                     if MODEL_ORIGIN == 'MADLC':
                         peaks = get_ma_pose(frame, config, sess, inputs, outputs)
-
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index, peaks, analysis_time))
 
         elif MODEL_ORIGIN == 'DLC-LIVE':
             dlc_live = load_dlc_live()
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     if not dlc_live.is_initialized:
                         peaks = dlc_live.init_inference(frame)
                     else:
                         peaks = dlc_live.get_pose(frame)
-
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index, peaks, analysis_time))
 
         elif MODEL_ORIGIN == 'DEEPPOSEKIT':
             predict_model = load_dpk()
             while True:
                 if input_q.full():
                     index, frame = input_q.get()
+                    start_time = time.time()
                     frame = frame[..., 1][..., None]
                     st_frame = np.stack([frame])
                     prediction = predict_model.predict(st_frame, batch_size=1, verbose=True)
                     peaks = prediction[0, :, :2]
-                    output_q.put((index, peaks))
+                    analysis_time = time.time() - start_time
+                    output_q.put((index,peaks,analysis_time))
+
+        elif MODEL_ORIGIN == 'SLEAP':
+            sleap_model = load_sleap()
+            while True:
+                if input_q.full():
+                    index, frame = input_q.get()
+                    start_time = time.time()
+                    input_frame = frame[:, :, ::-1]
+                    #this is weird, but without it, it does not seem to work...
+                    frames = np.array([input_frame])
+                    prediction = sleap_model.predict(frames[[0]], batch_size=1)
+                    #check if this is multiple animal instances or single animal model
+                    if  sleap_model.name == 'single_instance_inference_model':
+                        #get predictions (wrap it again, so the behavior is the same for both model types)
+                        peaks = np.array([prediction['peaks'][0, :]])
+                    else:
+                        peaks = prediction['instance_peaks'][0, :]
+                    analysis_time = time.time() - start_time
+                    output_q.put((index,peaks,analysis_time))
         else:
             raise ValueError(f'Model origin {MODEL_ORIGIN} not available.')
 
@@ -360,8 +388,12 @@ def get_frames(self) -> tuple:
         c_frames, d_maps, i_frames = self._camera_manager.get_frames()
         for camera in c_frames:
             c_frames[camera] = np.asanyarray(c_frames[camera])
+            if CROP:
+                c_frames[camera] = c_frames[camera][CROP_Y[0]:CROP_Y[1],CROP_X[0]:CROP_X[1]].copy()
+
         for camera in i_frames:
             i_frames[camera] = np.asanyarray(i_frames[camera])
+
         return c_frames, d_maps, i_frames
 
     def input_frames_for_analysis(self, frames: tuple, index: int):
@@ -379,9 +411,9 @@ def input_frames_for_analysis(self, frames: tuple, index: int):
                     frame_time = time.time()
                     self._multiprocessing[camera]['input'].put((index, frame))
                     if d_maps:
-                        self.store_frames(camera, frame, d_maps[camera], frame_time)
+                        self.store_frames(camera, frame, d_maps[camera], frame_time, index)
                     else:
-                        self.store_frames(camera, frame, None, frame_time)
+                        self.store_frames(camera, frame, None, frame_time, index)
 
     def get_analysed_frames(self) -> tuple:
         """
@@ -394,19 +426,24 @@ def get_analysed_frames(self) -> tuple:
         if self._dlc_running:
             analysed_frames = {}
             analysis_time = None
-            frame_width, frame_height = RESOLUTION
+
+            if CROP:
+                frame_height = CROP_Y[1] - CROP_Y[0]
+                frame_width = CROP_X[1] - CROP_X[0]
+            else:
+                frame_width,frame_height = RESOLUTION
+
             for camera in self._multiprocessing:
                 if self._multiprocessing[camera]['output'].full():
                     if self._start_time is None:
                         self._start_time = time.time()  # getting the first frame here
 
                     # Getting the analysed data
-                    analysed_index, peaks = self._multiprocessing[camera]['output'].get()
+                    analysed_index, peaks, analysis_time = self._multiprocessing[camera]['output'].get()
                     skeletons = calculate_skeletons(peaks, ANIMALS_NUMBER)
                     print('', end='\r', flush=True)  # this is the line you should not remove
-                    analysed_frame, depth_map, input_time = self.get_stored_frames(camera)
-                    analysis_time = time.time() - input_time
-
+                    analysed_frame , depth_map, input_time = self.get_stored_frames(camera, analysed_index)
+                    delay_time = time.time() - input_time
                     # Calculating FPS and plotting the data on frame
                     self.calculate_fps(analysis_time if analysis_time != 0 else 0.01)
                     frame_time = time.time() - self._start_time
@@ -419,8 +456,11 @@ def get_analysed_frames(self) -> tuple:
                         self._experiment_running = False
 
                     if self._experiment_running and not self._experiment.experiment_finished:
-                        for skeleton in skeletons:
-                            self._experiment.check_skeleton(analysed_image, skeleton)
+                        if ANIMALS_NUMBER > 1 and not FLATTEN_MA and not PASS_SEPARATE:
+                            self._experiment.check_skeleton(analysed_image,skeletons)
+                        else:
+                            for skeleton in skeletons:
+                                self._experiment.check_skeleton(analysed_image, skeleton)
 
                     # Gathering data as pd.Series for output
                     if self._data_output:
@@ -430,23 +470,30 @@ def get_analysed_frames(self) -> tuple:
                     analysed_frames[camera] = analysed_image
             return analysed_frames, analysis_time
 
-    def store_frames(self, camera: str, c_frame, d_map, frame_time: float):
+    def store_frames(self, camera: str, c_frame, d_map, frame_time: float, index: int):
         """
-        Store frames currently sent for analysis
+        Store frames currently sent for analysis in index based dictionary
         :param camera: camera name
         :param c_frame: color frame
         :param d_map: depth map
         :param frame_time: inputting time of frameset
+        :param index: index of frame that is currently analysed
         """
-        self._stored_frames[camera] = c_frame, d_map, frame_time
+        if camera in self._stored_frames.keys():
+            self._stored_frames[camera][index] = c_frame, d_map, frame_time
+
+        else:
+            self._stored_frames[camera] = {}
+            self._stored_frames[camera][index] = c_frame, d_map, frame_time
 
-    def get_stored_frames(self, camera: str):
+    def get_stored_frames(self, camera: str, index: int):
         """
-        Retrieve frames currently sent for analysis
+        Retrieve frames currently sent for analysis, retrieved frames will be removed (popped) from the dictionary
         :param camera: camera name
+        :param index: index of analysed frame
         :return:
         """
-        c_frame, d_map, frame_time = self._stored_frames.get(camera)
+        c_frame, d_map, frame_time = self._stored_frames[camera].pop(index, None)
         return c_frame, d_map, frame_time
 
     def convert_depth_map_to_image(self, d_map):

Readme.md

@@ -11,24 +11,37 @@
 
 DeepLabStream is a python based multi-purpose tool that enables the realtime tracking and manipulation of animals during ongoing experiments.
 Our toolbox was orginally adapted from the previously published [DeepLabCut](https://github.com/AlexEMG/DeepLabCut) ([Mathis et al., 2018](https://www.nature.com/articles/s41593-018-0209-y)) and expanded on its core capabilities, but is now able to utilize a variety of different network architectures for online pose estimation
- ([DLC + maDLC](https://github.com/AlexEMG/DeepLabCut), [DLC-Live](https://github.com/DeepLabCut/DeepLabCut-live), [DeepPosekit's](https://github.com/jgraving/DeepPoseKit) StackedDenseNet, StackedHourGlass and [LEAP](https://github.com/murthylab/sleap)).
+ ([SLEAP](https://github.com/murthylab/sleap), [DLC-Live](https://github.com/DeepLabCut/DeepLabCut-live), [DeepPosekit's](https://github.com/jgraving/DeepPoseKit) StackedDenseNet, StackedHourGlass and [LEAP](https://github.com/murthylab/sleap)).
+
 DeepLabStreams core feature is the utilization of real-time tracking to orchestrate closed-loop experiments. This can be achieved using any type of camera-based video stream (incl. multiple streams).  It enables running experimental protocols that are dependent on a constant stream of bodypart positions and feedback activation of several input/output devices. It's capabilities range from simple region of interest (ROI) based triggers to headdirection or behavior dependent stimulation.
 
 ![DLS_Stim](docs/DLSSTim_example.gif)
 
+## New features:
+
+#### 02/2021: Multiple Animal Experiments (Pre-release): Full [SLEAP](https://github.com/murthylab/sleap) integration (Full release coming soon!)
+
+- Updated [Installation](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki/Installation-&-Testing) (for SLEAP support)
+- Single Instance and Multiple Instance models (TopDown & BottomUp) integration (SLEAP v1.1.0a10), more coming soon!
+- [New Multiple Animal Experiment Tutorial](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki/Multiple-Animal-Experiments)
+
+#### 01/2021: DLStream was published in [Communications Biology](https://www.nature.com/articles/s42003-021-01654-9)
+
+#### 12/2021: New pose estimation model integration ([DLC-Live](https://github.com/DeepLabCut/DeepLabCut-live)) and pre-release of further integration ([DeepPosekit's](https://github.com/jgraving/DeepPoseKit) StackedDenseNet, StackedHourGlass and [LEAP](https://github.com/murthylab/sleap))
+
 ## Quick Reference:
 
- ### Check out or wiki: [DLStream Wiki](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki)
+ #### Check out or wiki: [DLStream Wiki](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki)
 
- ### Read the preprint: [Schweihoff et al, 2019](https://doi.org/10.1101/2019.12.20.884478).
+ #### Read the paper: [Schweihoff, et al. 2021](https://www.nature.com/articles/s42003-021-01654-9)
 
- ### Contributing
+ #### Contributing
 
 If you have feature requests or questions regarding the design of experiments join our [slack group](https://join.slack.com/t/dlstream/shared_invite/zt-jpy2olk1-CuJu0ZylGg_SLbO7zBkcrg)!
 
 We are constantly working to update and increase the capabilities of DLStream. 
 We welcome all feedback and input from your side.
-Also, do not hesitate to contact us for collaborations.
+
 
  ### 1. [Updated Installation & Testing](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki/Installation-&-Testing)
 
@@ -49,7 +62,7 @@ Also, do not hesitate to contact us for collaborations.
  ### 7. [Adapting an existing experiment to your own needs](https://github.com/SchwarzNeuroconLab/DeepLabStream/wiki/Adapting-an-existing-experiment-to-your-own-needs)
 
 
- 
+
 ### How to use DeepLabStream
 
 Just run 
@@ -110,7 +123,12 @@ If you encounter any issues or errors, you can check out the wiki article ([Help
 
 ## References:
 
-If you use this code or data please cite [Schweihoff et al, 2019](https://doi.org/10.1101/2019.12.20.884478).
+If you use this code or data please cite:
+
+
+Schweihoff, J.F., Loshakov, M., Pavlova, I. et al. DeepLabStream enables closed-loop behavioral experiments using deep learning-based markerless, real-time posture detection. 
+
+Commun Biol 4, 130 (2021). https://doi.org/10.1038/s42003-021-01654-9
 
 ## License
 This project is licensed under the GNU General Public License v3.0. Note that the software is provided "as is", without warranty of any kind, expressed or implied. 

app.py

@@ -11,13 +11,13 @@
 import cv2
 
 from DeepLabStream import DeepLabStream, show_stream
+from utils.generic import MissingFrameError
 from utils.configloader import MULTI_CAM, STREAMS, RECORD_EXP
 from utils.gui_image import QFrame, ImageWindow, emit_qframes
 
-from PyQt5.QtCore import QThread
-from PyQt5.QtWidgets import QPushButton, QApplication, QWidget, QGridLayout
-from PyQt5.QtGui import QIcon
-
+from PySide2.QtCore import QThread
+from PySide2.QtWidgets import QPushButton, QApplication, QWidget, QGridLayout
+from PySide2.QtGui import QIcon
 
 # creating a complete thread process to work in the background
 class AThread(QThread):
@@ -42,7 +42,16 @@ def run(self):
         Infinite loop with all the streaming, analysis and recording logic
         """
         while self.threadactive:
-            all_frames = stream_manager.get_frames()
+            try:
+                all_frames = stream_manager.get_frames()
+            except MissingFrameError as e:
+                """catch missing frame, stop Thread and save what can be saved"""
+                print(*e.args, '\nShutting down DLStream and saving data...')
+                stream_manager.finish_streaming()
+                stream_manager.stop_cameras()
+                self.stop()
+                break
+
             color_frames, depth_maps, infra_frames = all_frames
 
             # writing the video