benchmarking with restart

Author: AlexEMG

benchmarking/run.py

@@ -22,12 +22,12 @@
 pixels = [2500, 10000, 40000, 160000, 320000, 640000]
 ind = 1
 
-for m in dog_models:
-    print("\n\nMODEL {:d} / 8".format(ind))
-    benchmark_model_by_size(m, dog_video, output=out_dir, n_frames=n_frames, pixels=pixels)
-    ind += 1
-
-for m in mouse_models:
-    print("\n\nMODEL {:d} / 8".format(ind))
-    benchmark_model_by_size(m, mouse_video, output=out_dir, n_frames=n_frames, pixels=pixels)
-    ind += 1
+for ind_m, m in enumerate(dog_models):
+    print("\n\nMODEL {:d} / 8".format(ind_m))
+    benchmark_model_by_size(m, dog_video, ind_m, out_dir=out_dir, n_frames=n_frames, pixels=pixels)
+
+offset=ind_m+1
+
+for ind_m, m in enumerate(mouse_models):
+    print("\n\nMODEL {:d} / 8".format(ind_m))
+    benchmark_model_by_size(m, mouse_video, ind_m + offset, out_dir=out_dir, n_frames=n_frames, pixels=pixels)

dlclive/bench.py

@@ -141,7 +141,16 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100
     return inf_times, resize*im_size[0] * resize*im_size[1], TFGPUinference
 
 
-def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=None, out_dir=None):
+def get_savebenchmarkfn(sys_info ,i, fn_ind, out_dir=None):
+    ''' get filename to save data (definitions see save_benchmark)'''
+    out_dir = out_dir if out_dir is not None else os.getcwd()
+    host_name, op_sys, host_python, dev = sys_info
+
+    base_name = "benchmark_{}_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind, i)
+    datafilename = out_dir + '/' + base_name
+    return datafilename
+
+def save_benchmark(sys_info, inf_times, pixels, i, fn_ind, TFGPUinference, model=None, out_dir=None,datafilename=None):
     """ Save benchmarking data with system information to a pickle file
 
     Parameters
@@ -152,7 +161,7 @@ def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=Non
         array of inference times generated by :func:`run_benchmark`
     pixels : float or :class:`numpy.ndarray`
         number of pixels for each benchmark run. If an array, each index corresponds to a row in inf_times
-    iter: integer
+    i: integer
         number of the specific instance of experiment (so every part is saved individually)
     TFGPUinference: bool
         flag if using tensorflow inference or numpy inference DLC model
@@ -168,7 +177,10 @@ def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=Non
         flag indicating successful save
     """
 
-    out_dir = out_dir if out_dir is not None else os.getcwd()
+    if datafilename is None:
+        #out_dir = out_dir if out_dir is not None else os.getcwd()
+        datafilename=get_savebenchmarkfn(sys_info ,iter, fn_ind, out_dir=out_dir)
+
     host_name, op_sys, host_python, dev = sys_info
     host_name = host_name.replace(" ", "")
 
@@ -181,14 +193,6 @@ def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=Non
         else:
             model_type = None
 
-    fn_ind = 0
-    base_name = "benchmark_{}_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind,iter)
-    while os.path.isfile(os.path.normpath(out_dir + '/' + base_name)):
-        fn_ind += 1
-        base_name = "benchmark_{}_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind,iter)
-
-    fn = os.path.normpath(out_dir)
-
     data = {'host_name' : host_name,
             'op_sys' : op_sys,
             'python' : host_python,
@@ -200,7 +204,7 @@ def save_benchmark(sys_info, inf_times, pixels, iter , TFGPUinference, model=Non
             'pixels' : pixels,
             'inference_times' : inf_times}
 
-    pickle.dump(data, open(os.path.normpath(out_dir + '/' + base_name), 'wb'))
+    pickle.dump(data, open(os.path.normpath(datafilename), 'wb'))
 
     return True
 
@@ -209,7 +213,7 @@ def read_pickle(filename):
     with open(filename, "rb") as handle:
         return pickle.load(handle)
 
-def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, resize=None, pixels=None, print_rate=False):
+def benchmark_model_by_size(model_path, video_path, fn_ind, out_dir=None, n_frames=10000, resize=None, pixels=None, print_rate=False):
     """Benchmark DLC model by image size
 
     Parameters
@@ -218,7 +222,9 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000,
         path to exported DLC model
     video_path : str
         path to video file
-    output : str, optional
+    fn_ind : integer
+        auxiliary variable for creating a unique identifier for saving the models
+    out_dir : str, optional
         directory to save data, will not save data if None, by default None
     n_frames : int, optional
         number of frames to run, by default 10000
@@ -228,7 +234,7 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000,
         list of pixel image sizes (as ints), can only use one of resize or pixels, if both specified will use pixels, by default None
     print_rate : bool, optional
         flag to print frame by frame inference rate, by default False
-    
+
     Example
     --------
     Linux/MacOs
@@ -252,21 +258,25 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000,
     print(resize)
     for i in range(len(resize)):
 
-        print("\nRun {:d} / {:d}\n".format(i+1, len(resize)))
+        sys_info = get_system_info()
+        #print("Your system info:", sys_info)
+        datafilename=get_savebenchmarkfn(sys_info ,i, fn_ind, out_dir=out_dir)
 
-        inf_times, pixels_out, TFGPUinference = run_benchmark(model_path,
-                                                                    video_path,
-                                                                    resize=resize[i],
-                                                                    pixels=pixels[i],
-                                                                    n_frames=n_frames,
-                                                                    print_rate=print_rate)
+        #Check if a part has already been complted?
+        if os.path.isfile(os.path.normpath(datafilename)):
+            print("\nAlready ran {:d} / {:d}\n".format(i+1, len(resize)))
+        else:
+            print("\nRun {:d} / {:d}\n".format(i+1, len(resize)))
 
-        #TODO: check if a part has already been complted?
+            inf_times, pixels_out, TFGPUinference = run_benchmark(model_path,
+                                                                        video_path,
+                                                                        resize=resize[i],
+                                                                        pixels=pixels[i],
+                                                                        n_frames=n_frames,
+                                                                        print_rate=print_rate)
 
-        ### saving results intermediately
-        sys_info = get_system_info()
-        #print("Your system info:", sys_info)
-        save_benchmark(sys_info, inf_times, pixels_out, i, TFGPUinference, model=os.path.basename(model_path), out_dir=output)
+            ### saving results intermediately
+            save_benchmark(sys_info, inf_times, pixels_out, i, fn_ind, TFGPUinference, model=os.path.basename(model_path), datafilename=datafilename)
 
 
 def main():