Fix/timing plots (#124)

* fix plotting to work with khronos style names

* fix comparison

* clean up command line interface

Author: nathanhhughes

eval/python/hydra_eval/commands/timing.py

@@ -1,6 +1,5 @@
 """Command to gather timing information."""
 
-import json
 import pathlib
 import sys
 
@@ -16,9 +15,16 @@ def cli():
 
 @cli.command(name="show")
 @click.argument("result_path", type=click.Path(exists=True))
-@click.option("-k", "--key", "keys", multiple=True, default=None)
+@click.option(
+    "-k",
+    "--key",
+    "keys",
+    multiple=True,
+    default=None,
+    help="optional regex filters to select relevant timers",
+)
 def show(result_path, keys):
-    """Print timing information for a single set of Hydra results."""
+    """Print timing distribution from Hydra's output at RESULT_PATH."""
     result_path = pathlib.Path(result_path).expanduser().absolute()
     durations = timing.collect_timing_info(result_path)
     if len(durations) == 0:
@@ -34,33 +40,67 @@ def show(result_path, keys):
 
 @cli.command(name="plot")
 @click.argument("result_path", type=click.Path(exists=True))
-@click.option("-t", "--show-trends", is_flag=True, default=False)
-@click.option("-k", "--key", "keys", multiple=True, default=("frontend", "backend"))
-def plot(result_path, show_trends, keys):
-    """Plot timing information for a single set of Hydra results."""
+@click.option(
+    "-k",
+    "--key",
+    "keys",
+    multiple=True,
+    default=("frontend.*", "backend.*"),
+    help="optional regex filters to select timers for each subplot",
+)
+def plot(result_path, keys):
+    """Plot timing distribution from Hydra's output at RESULT_PATH."""
     result_path = pathlib.Path(result_path).expanduser().absolute()
     durations = timing.collect_timing_info(result_path)
     if len(durations) == 0:
         click.secho(f"Result path {result_path} does not contain timing info", fg="red")
         sys.exit(1)
 
-    if show_trends:
-        timing.plot_trends(durations, keys=keys)
-    else:
-        timing.plot_durations(durations, keys=keys)
+    timing.plot_durations(durations, keys=keys)
+
+
+@cli.command(name="timeline")
+@click.argument("result_path", type=click.Path(exists=True))
+@click.option(
+    "-k",
+    "--key",
+    "keys",
+    multiple=True,
+    default=("frontend.*", "backend.*"),
+    help="optional regex filters to select timers to plot",
+)
+def timeline(result_path, keys):
+    """Plot timing information against time from Hydra's output at RESULT_PATH."""
+    result_path = pathlib.Path(result_path).expanduser().absolute()
+    durations = timing.collect_timing_info(result_path)
+    if len(durations) == 0:
+        click.secho(f"Result path {result_path} does not contain timing info", fg="red")
+        sys.exit(1)
+
+    timing.plot_trends(durations, keys=keys)
 
 
 @cli.command(name="compare")
 @click.argument("results", nargs=-1, type=click.Path(exists=True))
-@click.option("-k", "--key", "keys", multiple=True, default=timing.DEFAULT_COMPARE_KEYS)
-@click.option("-c", "--config", default=None)
-def compare(results, keys, config):
-    """Plot timing comparison between different runs of Hydra."""
+@click.option(
+    "-k",
+    "--key",
+    "keys",
+    multiple=True,
+    default=("frontend/spin", "backend/spin"),
+    help="timer names to use for comparison",
+)
+@click.option(
+    "--use-bar",
+    is_flag=True,
+    default=False,
+    help="use a bar plot instead of a boxen plot",
+)
+def compare(results, keys, use_bar):
+    """Plot timing comparison between different runs of Hydra from RESULTS."""
     if len(results) == 0:
         return
 
     paths = [pathlib.Path(x).expanduser().absolute() for x in results]
     results = {path.stem: timing.collect_timing_info(path) for path in paths}
-
-    plot_config = {} if config is None else json.loads(config)
-    timing.plot_comparison(results, plot_config, keys)
+    timing.plot_comparison(results, keys, use_bars=use_bar)

eval/python/hydra_eval/timing.py

@@ -1,6 +1,7 @@
 """Some small helpers for getting timing information."""
 
 import logging
+import pathlib
 import re
 
 import matplotlib.pyplot as plt
@@ -10,12 +11,6 @@
 import rich.console
 import seaborn as sns
 
-DEFAULT_FOLDERS = ["frontend", "backend", "lcd", "timing"]
-FRONTEND_TIMERS = ["frontend/spin"]
-BACKEND_TIMERS = ["backend/spin"]
-OBJECT_TIMERS = ["frontend/objects_detection", "frontend/objects_graph_update"]
-PLACE_TIMERS = ["frontend/detect_gvd", "frontend/update_gvd_places"]
-ROOM_TIMERS = ["backend/room_detection"]
 DEFAULT_COMPARE_KEYS = [
     "frontend/object_detection",
     "gvd/extract_graph",
@@ -31,11 +26,12 @@ def _get_time_array_from_log(filename):
     return arr
 
 
-def _get_filenames(result_path, subdir):
-    if not (result_path / subdir).exists():
+def _get_filenames(result_path):
+    result_path = pathlib.Path(result_path)
+    if not result_path.exists():
         return []
 
-    return [f for f in (result_path / subdir).iterdir() if f.match("*timing_raw.csv")]
+    return [f for f in result_path.rglob("*timing_raw.csv")]
 
 
 def _get_timer_name(path):
@@ -64,22 +60,20 @@ def _get_longform_df(durations, filter_func=None):
             continue
 
         times = np.squeeze(info[:, 1])
-        part_name = name.split("/")[1]
         data = np.concatenate((data, times), axis=None)
-        names += [part_name] * len(times)
+        names += [name] * len(times)
 
     if len(names) == 0:
         return None
 
     return pd.DataFrame({"Name": names, "Elapsed Time [s]": data})
 
 
-def collect_timing_info(path, default_folders=None):
+def collect_timing_info(path, folders_to_use=None):
     """Collect timing info for a specfic path."""
-    timing_files = []
-    folders_to_use = default_folders or DEFAULT_FOLDERS
-    for folder in folders_to_use:
-        timing_files += _get_filenames(path, folder)
+    timing_files = _get_filenames(path)
+    if folders_to_use:
+        timing_files = [x for x in timing_files if x.parent in folders_to_use]
 
     return {_get_timer_name(f): _get_time_array_from_log(f) for f in timing_files}
 
@@ -139,30 +133,30 @@ def _get_stat_str(stat):
     console.print(table)
 
 
-def plot_durations(durations, keys=["frontend", "backend"], rt_threshold=None):
+def plot_durations(durations, keys):
     """Make a plot of durations."""
     sns.set()
     sns.set_style("whitegrid")
     sns.set_context("notebook")
     fig, ax = plt.subplots(len(keys), 1, squeeze=False)
 
     for idx, key in enumerate(keys):
-        ax[idx][0].set_title(f"{key.capitalize()} Timing Distributions")
-        df = _get_longform_df(durations, lambda x: key not in x)
+        matcher = re.compile(key)
+        ax[idx][0].set_title(f"{key} Timing Distributions")
+        df = _get_longform_df(durations, lambda x: matcher.match(x))
         if df is None:
             continue
 
         sns.boxenplot(data=df, x="Name", y="Elapsed Time [s]", ax=ax[idx][0])
         lax = ax[idx][0]
         lax.set_xticks(lax.get_xticks(), lax.get_xticklabels(), rotation=30, ha="right")
-        _draw_realtime_threshold(lax, rt_threshold)
 
     fig.set_size_inches([14, 12 * len(keys)])
     fig.tight_layout()
     plt.show()
 
 
-def plot_trends(durations, keys=["frontend", "backend"], rt_threshold=None):
+def plot_trends(durations, keys):
     """Make a plot of durations."""
     sns.set()
     sns.set_style("whitegrid")
@@ -171,8 +165,9 @@ def plot_trends(durations, keys=["frontend", "backend"], rt_threshold=None):
 
     for idx, key in enumerate(keys):
         added_plot = False
+        matcher = re.compile(key)
         for name, time_array in durations.items():
-            if key not in name:
+            if not matcher.match(name):
                 continue
 
             assert len(time_array.shape) == 2 and time_array.shape[1] == 2
@@ -183,60 +178,51 @@ def plot_trends(durations, keys=["frontend", "backend"], rt_threshold=None):
             ax[idx][0].plot(to_plot[:, 0], to_plot[:, 1], label=name)
             added_plot = True
 
-        ax[idx][0].set_title(f"{key.capitalize()} Timing Trends")
+        ax[idx][0].set_title(f"{key} Timing Trends")
         ax[idx][0].set_xlabel("Timestamp [s]")
         ax[idx][0].set_ylabel("Elapsed Time [s]")
         if added_plot:
-            _draw_realtime_threshold(ax[idx][0], rt_threshold)
             ax[idx][0].legend()
 
     fig.set_size_inches([16, 8 * len(keys)])
     plt.show()
 
 
-def plot_comparison(results, plot_config, keys, use_bars=False, rt_threshold=None):
+def plot_comparison(results, keys, use_bars=False):
     """Plot timing comparison for different results directories."""
+    sns.set()
+    sns.set_style("whitegrid")
+    sns.set_context("notebook")
+
     data = np.array([])
     labels = []
     result_set = []
 
     keys = [x.replace("/", "_") for x in keys]
     for stem, result in results.items():
-        result_label = plot_config.get("result_map", {}).get(stem, stem.upper())
         for key in keys:
             if key not in result:
                 continue
 
-            # TODO(nathan) remap key to actual name
-            # TODO(nathan) add result key
-            key_label = plot_config.get("key_map", {}).get(key, key)
-
             num_values = len(result[key])
-            labels += num_values * [key_label]
-            result_set += num_values * [result_label]
+            data = np.hstack((data, np.squeeze(result[key][:, 1])))
+            labels += num_values * [key]
+            result_set += num_values * [stem.upper()]
 
-    sns.set()
-    sns.set_style("white")
-    sns.set_context("poster", font_scale=1.7, rc={"lines.linewidth": 2.0})
-
-    fig, ax = plt.subplots()
     value_key = "Elapsed Time [s]"
-    timer_key = plot_config.get("xlabel", "Timer Name")
+    timer_key = "Timer Name"
     result_key = "Result Name"
 
+    fig, ax = plt.subplots()
     df = pd.DataFrame({value_key: data, timer_key: labels, result_key: result_set})
     if use_bars:
-        sns.barplot(x=timer_key, y=value_key, hue=result_key, data=df)
+        sns.barplot(x=timer_key, y=value_key, hue=result_key, data=df, ax=ax)
     else:
-        sns.boxenplot(x=timer_key, y=value_key, hue=result_key, data=df)
-
-    if "title" in plot_config:
-        ax.set_title(plot_config["title"])
+        sns.boxenplot(x=timer_key, y=value_key, hue=result_key, data=df, ax=ax)
 
-    _draw_realtime_threshold(ax, rt_threshold)
     ax.legend()
+    ax.set_xticks(ax.get_xticks(), ax.get_xticklabels(), rotation=30, ha="right")
 
-    sns.despine()
     fig.set_size_inches([14, 12])
     fig.tight_layout()
     plt.show()