diff --git a/benchmarks/benchmark_and_profile.py b/benchmarks/benchmark_and_profile.py new file mode 100644 index 0000000..5ec3f3d --- /dev/null +++ b/benchmarks/benchmark_and_profile.py @@ -0,0 +1,293 @@ +import inspect +from timeit import default_timer +import numpy as np +from skimage import ( + draw, + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + data, + color, +) +import pandas as pd +import plotly.express as px +from line_profiler import LineProfiler, show_func +import io +import re + +profile = LineProfiler() + + +l1 = 1000 + +img = np.random.randint(256, size=(l1, l1), dtype=np.uint8) +img_rgb = np.random.randint(256, size=(l1, l1, 3), dtype=np.uint8) +img_binary = data.binary_blobs(length=l1, volume_fraction=0.3).astype(np.uint8) +img2 = data.binary_blobs(length=l1, volume_fraction=0.3, seed=10).astype(np.uint8) + + +parameters = { + "match_histograms": dict(reference=img2), + "cycle_spin": dict(func=restoration.denoise_wavelet, max_shifts=4), + "gabor": dict(frequency=0.5), + "denoise_tv_bregman": dict(weight=1.0), + "apply_hysteresis_threshold": dict(low=0.4, high=0.6), + "hough_circle": dict(radius=10), + "rescale": dict(scale=1.1), + "rotate": dict(angle=10), + "block_reduce": dict(block_size=(2, 2)), + "flood": dict(seed_point=(0, 0)), + "flood_fill": dict(seed_point=(0, 0), new_value=2), + "join_segmentations": dict(s2=img2), + "inpaint_biharmonic": dict(mask=img2), + "contingency_table": dict(im_test=img2), + "hausdorff_distance": dict(image1=img2), + "compare_images": dict(image2=img2), + "mean_squared_error": dict(image1=img2), + "normalized_root_mse": dict(image_test=img2), + "peak_signal_noise_ratio": dict(image_test=img2), + "structural_similarity": dict(im2=img2), + "variation_of_information": dict(image1=img2), + "optical_flow_tvl1": dict(moving_image=img2), + "phase_cross_correlation": dict(moving_image=img2), + "threshold_local": dict(block_size=l1 // 8 if (l1 // 8) % 2 == 1 else l1 // 8 + 1), + "downscale_local_mean": dict(factors=(2,) * img.ndim), + "difference_of_gaussians": dict(low_sigma=1), + "find_contours": dict(level=0.5), + "h_maxima": dict(h=10), + "h_minima": dict(h=10), +} + +need_binary_image = [ + "convex_hull_object", + "convex_hull_image", + "hausdorff_distance", + "remove_small_holes", + "remove_small_objects", +] + +need_rgb_image = ["quickshift", "rgb2lab", "rgb2xyz", "xyz2lab"] + +skip_functions = [ + "integrate", + "hough_circle_peaks", + "hough_line_peaks", + "ransac", + "window", + "hough_ellipse", + "view_as_blocks", + "view_as_windows", + "apply_parallel", + "regular_grid", + "regular_seeds", + "estimate_transform", + "matrix_transform", + "draw_haar_like_feature", + "corner_subpix", + "calibrate_denoiser", + "ball", + "cube", + "diamond", + "disk", + "octagon", + "octahedron", + "rectangle", + "square", + "star", + "hessian_matrix_eigvals", + "hessian_matrix_det", + "structure_tensor_eigvals", +] + +slow_functions = ["iradon_sart", "chan_vese", "inpaint_biharmonic", "quickshift"] + + +def only_one_nondefault(args): + """ + Returns True if the function has only one non-keyword parameter, + False otherwise. + """ + defaults = 0 if args.defaults is None else len(args.defaults) + if len(args.args) >= 1 and (len(args.args) - defaults <= 1): + return True + else: + return False + + +def _strip_docstring(func_str, max_len=60): + """remove docstring from code block so that is does not overflow the plotly + hover. + """ + line_number = len([m.start() for m in re.finditer("\n", func_str)]) + if line_number < max_len: + res = func_str + else: + try: + open_docstring, end_docstring = [ + m.start() for m in re.finditer('"""', func_str) + ] + res = func_str[:open_docstring] + func_str[end_docstring + 3 :] + except ValueError: + res = func_str + return res + + +def run_benchmark( + img, + img_binary, + img_rgb, + module_list=[ + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + color, + ], + skip_functions=[], +): + times = {} + + functions = [] + for submodule in module_list: + functions += inspect.getmembers(submodule, inspect.isfunction) + non_tested_functions = [] + + for function in functions: + args = inspect.getfullargspec(function[1]) + only_one_argument = only_one_nondefault(args) + if function[0] in skip_functions: + continue + if only_one_argument or function[0] in parameters: + params = parameters[function[0]] if function[0] in parameters else {} + try: + if function[0] in need_binary_image: + im = img_binary + elif function[0] in need_rgb_image: + im = img_rgb + else: + im = img + profile.add_function(function[1]) + start = default_timer() + profile.runcall(function[1], im, **params) + end = default_timer() + times[function[0]] = end - start + except: + non_tested_functions.append(function[0]) + else: + non_tested_functions.append(function[0]) + return times, non_tested_functions + + +# ----------------------- Run benchmark ------------------------- +if __name__ == "__main__": + + module_list = [ + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + color, + ] + + # use skip_functions=skip_functions + slow_functions for a first test + # since slow functions are really slow + times, non_tested_functions = run_benchmark( + img, img_binary, img_rgb, skip_functions=skip_functions + ) + function_names = sorted(times, key=times.get) + sorted_times = sorted(times.values()) + + # ----------------------- Print results ------------------------- + df = [] + + print("Functions which could not be tested") + print("=" * 70) + + for func in non_tested_functions: + print(func) + + print("\n") + print("Sorted by increasing execution time") + print("=" * 70) + + for func_name, t in zip(function_names, sorted_times): + print(func_name, t) + + print("\n") + print("Sorted by subpackage") + print("=" * 70) + + for submodule in module_list: + print("\n") + print(submodule.__name__) + print("-" * 70) + for func in inspect.getmembers(submodule, inspect.isfunction): + if func[0] in times: + print(func[0], times[func[0]]) + df.append( + { + "module": submodule.__name__, + "function": func[0], + "time": times[func[0]], + } + ) + + # ------------------- Retrieve results from line profiler -------- + line_stats = profile.get_stats() + df = pd.DataFrame(df) + df = df.sort_values(by=["time"]) + df["timings"] = "" + for (fn, lineno, name), timings in line_stats.timings.items(): + output = io.StringIO() + show_func( + fn, + lineno, + name, + line_stats.timings[fn, lineno, name], + line_stats.unit, + stream=output, + ) + dump = _strip_docstring(output.getvalue(), max_len=20).replace("\n", "
") + df.loc[df.function == name, "timings"] = dump + + # ----------------------- Display results ------------------------- + # threshold = np.quantile(df['time'], 0.85) + # df['text'] = df['function'] + # df['text'][df['time'] < threshold] = '' + fig = px.scatter( + df, + x="function", + y="time", + color="module", + log_y=True, + hover_data=["timings"], + template="presentation", + ) + fig.update_xaxes(tickfont_size=8) + fig.update_traces(textposition="top left") + fig.update_layout( + title_text=f"Execution time for a {l1}x{l1} image", + hoverlabel_align="left", + hovermode="closest", + ) + config = {"toImageButtonOptions": {"height": None, "width": None,}} + fig.show(config=config) diff --git a/benchmarks/benchmark_suite.py b/benchmarks/benchmark_suite.py new file mode 100644 index 0000000..0ab8456 --- /dev/null +++ b/benchmarks/benchmark_suite.py @@ -0,0 +1,240 @@ +import inspect +from timeit import default_timer +import numpy as np +from skimage import ( + draw, + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + util, + data, +) +import pandas as pd +import plotly.express as px + + + +l1 = 1000 + +img = np.random.randint(256, size=(l1, l1), dtype=np.uint8) +img_rgb = np.random.randint(256, size=(l1, l1, 3), dtype=np.uint8) +img_binary = data.binary_blobs(length=l1, volume_fraction=0.3).astype(np.uint8) +img2 = data.binary_blobs(length=l1, volume_fraction=0.3, seed=10).astype(np.uint8) + + +parameters = { + 'match_histograms': dict(reference=img2), + 'cycle_spin': dict(func=restoration.denoise_wavelet, max_shifts=4), + 'gabor': dict(frequency=0.5), + 'denoise_tv_bregman': dict(weight=1.0), + 'apply_hysteresis_threshold': dict(low=0.4, high=0.6), + 'hough_circle': dict(radius=10), + 'rescale': dict(scale=1.1), + 'rotate': dict(angle=10), + 'block_reduce': dict(block_size=(2, 2)), + 'flood': dict(seed_point=(0, 0)), + 'flood_fill': dict(seed_point=(0, 0), new_value=2), + 'join_segmentations': dict(s2=img2), + 'inpaint_biharmonic': dict(mask=img2), + 'contingency_table': dict(im_test=img2), + 'hausdorff_distance': dict(image1=img2), + 'compare_images': dict(image2=img2), + 'mean_squared_error': dict(image1=img2), + 'normalized_root_mse': dict(image_test=img2), + 'peak_signal_noise_ratio': dict(image_test=img2), + 'structural_similarity': dict(im2=img2), + 'variation_of_information': dict(image1=img2), + 'optical_flow_tvl1': dict(moving_image=img2), + 'phase_cross_correlation': dict(moving_image=img2), + 'threshold_local': dict(block_size=l1 // 8 if (l1 //8) % 2 == 1 else l1 // 8 + 1), + 'downscale_local_mean': dict(factors=(2,)*img.ndim), + 'difference_of_gaussians': dict(low_sigma=1), + 'find_contours': dict(level=0.5), + 'h_maxima': dict(h=10), + 'h_minima': dict(h=10), + } + +need_binary_image = [ + 'convex_hull_object', + 'convex_hull_image', + 'hausdorff_distance', + 'remove_small_holes', + 'remove_small_objects', + ] + +need_rgb_image = ['quickshift'] + +skip_functions = [ + 'integrate', + 'hough_circle_peaks', + 'hough_line_peaks', + 'ransac', + 'window', + 'hough_ellipse', + 'view_as_blocks', + 'view_as_windows', + 'apply_parallel', + 'regular_grid', + 'regular_seeds', + 'estimate_transform', + 'matrix_transform', + 'draw_haar_like_feature', + 'corner_subpix', + 'calibrate_denoiser', + 'ball', + 'cube', + 'diamond', + 'disk', + 'octagon', + 'octahedron', + 'rectangle', + 'square', + 'star', + 'hessian_matrix_eigvals', + 'hessian_matrix_det', + 'structure_tensor_eigvals', + ] + +slow_functions = [ + 'iradon_sart', + 'chan_vese', + 'inpaint_biharmonic', + 'quickshift' + ] + +def only_one_nondefault(args): + """ + Returns True if the function has only one non-keyword parameter, + False otherwise. + """ + defaults = 0 if args.defaults is None else len(args.defaults) + if len(args.args) >= 1 and (len(args.args) - defaults <= 1): + return True + else: + return False + + +def run_benchmark( + img, + img_binary, + img_rgb, + module_list=[ + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + util, + ], + skip_functions=[], +): + times = {} + + functions = [] + for submodule in module_list: + functions += inspect.getmembers(submodule, inspect.isfunction) + non_tested_functions = [] + + for function in functions: + args = inspect.getfullargspec(function[1]) + only_one_argument = only_one_nondefault(args) + if function[0] in skip_functions: + continue + if only_one_argument or function[0] in parameters: + params = parameters[function[0]] if function[0] in parameters else {} + try: + start = default_timer() + if function[0] in need_binary_image: + im = img_binary + elif function[0] in need_rgb_image: + im = img_rgb + else: + im = img + function[1](im, **params) + end = default_timer() + times[function[0]] = end - start + except: + non_tested_functions.append(function[0]) + else: + non_tested_functions.append(function[0]) + return times, non_tested_functions + + +# ----------------------- Run benchmark ------------------------- +if __name__ == '__main__': + + module_list = [ + exposure, + feature, + filters, + measure, + metrics, + morphology, + registration, + restoration, + segmentation, + transform, + util, + ] + + # use skip_functions=skip_functions + slow_functions for a first test + # since slow functions are really slow + times, non_tested_functions = run_benchmark( + img, img_binary, img_rgb, skip_functions=skip_functions + ) + function_names = sorted(times, key=times.get) + sorted_times = sorted(times.values()) + + # ----------------------- Print results ------------------------- + df = [] + + print("Functions which could not be tested") + print("=" * 70) + + for func in non_tested_functions: + print(func) + + print("\n") + print("Sorted by increasing execution time") + print("=" * 70) + + for func_name, t in zip(function_names, sorted_times): + print(func_name, t) + + print("\n") + print("Sorted by subpackage") + print("=" * 70) + + for submodule in module_list: + print("\n") + print(submodule.__name__) + print("-" * 70) + for func in inspect.getmembers(submodule, inspect.isfunction): + if func[0] in times: + print(func[0], times[func[0]]) + df.append({'module': submodule.__name__, 'function': func[0], 'time': times[func[0]]}) + + # ----------------------- Display results ------------------------- + df = pd.DataFrame(df) + df = df.sort_values(by=['time']) + threshold = np.quantile(df['time'], 0.85) + df['text'] = df['function'] + df['text'][df['time'] < threshold] = '' + fig = px.scatter(df, x='function', y='time', text='text', color='module', log_y=True) + fig.update_xaxes(tickfont_size=8) + fig.update_traces(textposition='top left') + fig.update_layout(title_text=f'Execution time for a {l1}x{l1} image') + config = {'toImageButtonOptions': { 'height': None, 'width': None, }} + fig.show(config=config) diff --git a/benchmarks/define_arguments.py b/benchmarks/define_arguments.py new file mode 100644 index 0000000..5ba7f43 --- /dev/null +++ b/benchmarks/define_arguments.py @@ -0,0 +1,86 @@ +from skimage import restoration, data +import numpy as np + +l1 = 1000 + +img = np.random.randint(256, size=(l1, l1), dtype=np.uint8) +img_rgb = np.random.randint(256, size=(l1, l1, 3), dtype=np.uint8) +img_binary = data.binary_blobs(length=l1, volume_fraction=0.3).astype(np.uint8) +img2 = data.binary_blobs(length=l1, volume_fraction=0.3, seed=10).astype(np.uint8) + + +parameters = { + 'match_histograms': dict(reference=img2), + 'cycle_spin': dict(func=restoration.denoise_wavelet, max_shifts=4), + 'gabor': dict(frequency=0.5), + 'denoise_tv_bregman': dict(weight=1.0), + 'apply_hysteresis_threshold': dict(low=0.4, high=0.6), + 'hough_circle': dict(radius=10), + 'rescale': dict(scale=1.1), + 'rotate': dict(angle=10), + 'block_reduce': dict(block_size=(2, 2)), + 'flood': dict(seed_point=(0, 0)), + 'flood_fill': dict(seed_point=(0, 0), new_value=2), + 'join_segmentations': dict(s2=img2), + 'inpaint_biharmonic': dict(mask=img2), + 'contingency_table': dict(im_test=img2), + 'hausdorff_distance': dict(image1=img2), + 'compare_images': dict(image2=img2), + 'mean_squared_error': dict(image1=img2), + 'normalized_root_mse': dict(image_test=img2), + 'peak_signal_noise_ratio': dict(image_test=img2), + 'structural_similarity': dict(im2=img2), + 'variation_of_information': dict(image1=img2), + 'optical_flow_tvl1': dict(moving_image=img2), + 'phase_cross_correlation': dict(moving_image=img2), + 'threshold_local': dict(block_size=l1 // 8 if (l1 //8) % 2 == 1 else l1 // 8 + 1), + 'downscale_local_mean': dict(factors=(2,)*img.ndim), + 'difference_of_gaussians': dict(low_sigma=1), + 'find_contours': dict(level=0.5), + 'h_maxima': dict(h=10), + 'h_minima': dict(h=10), + } + +need_binary_image = [ + 'convex_hull_object', + 'convex_hull_image', + 'hausdorff_distance', + 'remove_small_holes', + 'remove_small_objects', + ] + +need_rgb_image = ['quickshift'] + +skip_functions = [ + 'integrate', + 'hough_circle_peaks', + 'hough_line_peaks', + 'ransac', + 'window', + 'hough_ellipse', + 'view_as_blocks', + 'view_as_windows', + 'apply_parallel', + 'regular_grid', + 'regular_seeds', + 'estimate_transform', + 'matrix_transform', + 'draw_haar_like_feature', + 'corner_subpix', + 'calibrate_denoiser', + 'ball', + 'cube', + 'diamond', + 'disk', + 'octagon', + 'octahedron', + 'rectangle', + 'square', + 'star' + ] + +slow_functions = [ + 'iradon_sart', + 'chan_vese', + 'inpaint_biharmonic', + ]