added benchmark folder, save intermediatly and save more metadata

Author: AlexEMG

benchmarking/howIbenchmarked.txt

@@ -0,0 +1,23 @@
+https://github.com/DeepLabCut/DeepLabCut-live/blob/master/docs/install_desktop.md
+
+Installation:
+```
+conda create -n dlc-live python=3.7 tensorflow-gpu==1.13.1 # if using GPU
+```
+
+From commit:
+https://github.com/DeepLabCut/DeepLabCut-live/commit/58636b2178a322f03c9084542a932ace39044815
+
+```
+conda activate dlc-live
+pip install git+https://github.com/AlexEMG/DeepLabCut-live.git
+pip install opencv-python
+```
+
+download data...
+
+go to /benchmarking/run.py 
+
+and update your datafolder to where the data is.
+
+run n_frames = 1000 on a CPU and 10k on GPU!

benchmarking/requirements.txt

@@ -0,0 +1,2 @@
+# NOTES, for benchmarking you also need:
+pip install py-cpuinfo==5.0

examples/run.py benchmarking/run.pyexamples/run.py renamed to benchmarking/run.py

@@ -5,15 +5,20 @@
 
 from dlclive import benchmark_model_by_size
 
-this_dir = os.path.dirname(os.path.realpath(__file__))
+# Update the datafolder to where the data is:
+datafolder='/media/alex/dropboxdisk/DLC-LiveBenchmarking/DLC-live-benchmarkingdata'
+
 
-dog_models = glob.glob(this_dir + '/dog/*[!avi]')
-dog_video = glob.glob(this_dir + '/dog/*.avi')[0]
-mouse_models = glob.glob(this_dir + '/mouse_lick/*[!avi]')
-mouse_video = glob.glob(this_dir + '/mouse_lick/*.avi')[0]
+dog_models = glob.glob(datafolder + '/dog/*[!avi]')
+dog_video = glob.glob(datafolder + '/dog/*.avi')[0]
+mouse_models = glob.glob(datafolder + '/mouse_lick/*[!avi]')
+mouse_video = glob.glob(datafolder + '/mouse_lick/*.avi')[0]
+
+this_dir = os.path.dirname(os.path.realpath(__file__))
+#storing results in /benchmarking/results: (for your PR)
 out_dir = os.path.normpath(this_dir + '/results')
 
-n_frames = 10000
+n_frames = 1000
 pixels = [2500, 10000, 40000, 160000, 320000, 640000]
 ind = 1
 

dlclive/bench.py

@@ -24,7 +24,7 @@
 
 def get_system_info():
     """ Return summary info for system running benchmark
-    
+
     Returns
     -------
     str
@@ -42,6 +42,8 @@ def get_system_info():
 
     op_sys = platform.platform()
     host_name = platform.node()
+
+    # A string giving the absolute path of the executable binary for the Python interpreter, on systems where this makes sense.
     if platform.system() == 'Windows':
         host_python = sys.executable.split(os.path.sep)[-2]
     else:
@@ -58,15 +60,15 @@ def get_system_info():
         dev_type = "GPU"
     else:
         from cpuinfo import get_cpu_info
-        dev = [get_cpu_info()['brand']]
+        dev = get_cpu_info() #[get_cpu_info()['brand']]
         dev_type = "CPU"
 
     return host_name, op_sys, host_python, (dev_type, dev)
 
 
 def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=10000, print_rate=False):
     """ Benchmark on inference times for a given DLC model and video
-    
+
     Parameters
     ----------
     model_path : str
@@ -81,7 +83,7 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100
         number of frames to run inference on, by default 10000
     print_rate : bool, optional
         flat to print inference rate frame by frame, by default False
-    
+
     Returns
     -------
     :class:`numpy.ndarray`
@@ -123,7 +125,7 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100
         start_pose = time.time()
         live.get_pose(frame)
         inf_times[i] = time.time() - start_pose
-        
+
         if print_rate:
             print("pose rate = {:d}".format(int(1 / inf_times[i])))
 
@@ -139,9 +141,9 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100
     return inf_times, resize*im_size[0] * resize*im_size[1], TFGPUinference
 
 
-def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_dir=None):
+def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=None, out_dir=None):
     """ Save benchmarking data with system information to a pickle file
-    
+
     Parameters
     ----------
     sys_info : tuple
@@ -150,13 +152,16 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_
         array of inference times generated by :func:`run_benchmark`
     pixels : float or :class:`numpy.ndarray`
         number of pixels for each benchmark run. If an array, each index corresponds to a row in inf_times
+    iter: integer
+        number of the specific instance of experiment (so every part is saved individually)
     TFGPUinference: bool
         flag if using tensorflow inference or numpy inference DLC model
     model: str, optional
         name of model
     out_dir : str, optional
         path to directory to save data. If None, uses pwd, by default None
-    
+
+
     Returns
     -------
     bool
@@ -166,6 +171,7 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_
     out_dir = out_dir if out_dir is not None else os.getcwd()
     host_name, op_sys, host_python, dev = sys_info
     host_name = host_name.replace(" ", "")
+
     model_type = None
     if model is not None:
         if 'resnet' in model:
@@ -176,10 +182,10 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_
             model_type = None
 
     fn_ind = 0
-    base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind)
+    base_name = "benchmark_{}_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind,iter)
     while os.path.isfile(os.path.normpath(out_dir + '/' + base_name)):
         fn_ind += 1
-        base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind)
+        base_name = "benchmark_{}_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind,iter)
 
     fn = os.path.normpath(out_dir)
 
@@ -198,10 +204,14 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_
 
     return True
 
+def read_pickle(filename):
+    """ Read the pickle file """
+    with open(filename, "rb") as handle:
+        return pickle.load(handle)
 
 def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, resize=None, pixels=None, print_rate=False):
     """Benchmark DLC model by image size
-    
+
     Parameters
     ----------
     model_path : str
@@ -232,24 +242,26 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000,
 
     ### initialize full inference times
 
-    inf_times = np.zeros((len(resize), n_frames))
-    pixels_out = np.zeros(len(resize))
-
+    #inf_times = np.zeros((len(resize), n_frames))
+    #pixels_out = np.zeros(len(resize))
+    print(resize)
     for i in range(len(resize)):
 
         print("\nRun {:d} / {:d}\n".format(i+1, len(resize)))
 
-        inf_times[i], pixels_out[i], TFGPUinference = run_benchmark(model_path,
+        inf_times, pixels_out, TFGPUinference = run_benchmark(model_path,
                                                                     video_path,
                                                                     resize=resize[i],
                                                                     pixels=pixels[i],
                                                                     n_frames=n_frames,
                                                                     print_rate=print_rate)
 
-    ### save results
+        #TODO: check if a part has already been complted?
 
-    sys_info = get_system_info()
-    save_benchmark(sys_info, inf_times, pixels_out, TFGPUinference, model=os.path.basename(model_path), out_dir=output)
+        ### saving results intermediately
+        sys_info = get_system_info()
+        #print("Your system info:", sys_info)
+        save_benchmark(sys_info, inf_times, pixels_out, i, TFGPUinference, model=os.path.basename(model_path), out_dir=output)
 
 
 def main():
@@ -274,4 +286,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

dlclive/version.py

@@ -8,5 +8,5 @@
 """
 
 
-__version__ = '0.0.b0'
+__version__ = '0.0.b1'
 VERSION = __version__

reinstall.sh

@@ -1,4 +1,3 @@
 pip uninstall deeplabcut-live
 python3 setup.py sdist bdist_wheel
-pip install dist/deeplabcut_live-0.0b0-py3-none-any.whl
-
+pip install dist/deeplabcut_live-0.0b1-py3-none-any.whl

setup.py

@@ -25,7 +25,7 @@
 
 setuptools.setup(
     name="deeplabcut-live",
-    version="0.0.b0",
+    version="0.0.b1",
     author="A. & M. Mathis Labs",
     author_email="[email protected]",
     description="Class to load exported DeepLabCut networks and perform pose estimation on single frames (from a camera feed)",