results: Some plot/table prettification and writing

Author: jonnor

TODO.md

@@ -65,8 +65,6 @@ Remove dots at the end. 2.2, 2.6, 2.16 etc
 - Fix Github link, to be to final branch
 - Make sure final page is EVEN number
 
-
-
 ### After report
 
 

microesc/report.py

@@ -87,17 +87,25 @@ def get_accuracies(confusions):
     assert len(accs) == 10, len(accs) 
     return pandas.Series(accs) 
 
-def plot_accuracy_comparison(experiments, ylim=(0.60, 0.80), figsize=(12, 4)):
+def plot_accuracy_comparison(experiments, ylim=(0.0, 1.0), figsize=(12, 4)):
 
     df = experiments.copy()
     df.index = experiments.nickname
     acc = df.confusions_test.apply(get_accuracies).T
-    fig, ax = plt.subplots(1, figsize=figsize)
+    fig, ax = plt.subplots(1, figsize=figsize, dpi=300)
 
     acc.boxplot(ax=ax)
 
+    # Mark SOTA models
+    ax.axhline(0.79, linestyle='dotted', color='green')
+    ax.axhline(0.83, linestyle='dotted', color='green')
+
+    # FIXME: better no-information rate
+    ax.axhline(0.10, linestyle='dotted', color='black')
+
     ax.set_ylabel('Accuracy')
     ax.set_ylim(ylim)
+    ax.set_yticks(numpy.arange(ylim[0], ylim[1], 0.1))
 
     #ax.set_xticks(experiments.nickname)
     #ax.set_xlabel('Model')
@@ -106,7 +114,6 @@ def plot_accuracy_comparison(experiments, ylim=(0.60, 0.80), figsize=(12, 4)):
 
 def plot_accuracy_vs_compute(experiments, ylim=(0.60, 0.80),
                                 perf_metric='utilization', figsize=(12,8)):
-    # TODO: color experiment groups
     # TODO: add error bars?
 
     acc = experiments.confusions_test.apply(get_accuracies).T
@@ -115,12 +122,23 @@ def plot_accuracy_vs_compute(experiments, ylim=(0.60, 0.80),
     numpy.testing.assert_allclose(df.test_acc_mean, df.accuracy)
     df['experiment'] = df.index
 
-    fig, ax = plt.subplots(1, figsize=figsize)
-    df.plot.scatter(ax=ax, x=perf_metric, y='accuracy', logx=True)
+    fig, ax = plt.subplots(1, figsize=figsize, dpi=300)
+    def get_color(idx, nick):
+        if nick.startswith('Stride-DS-') and not nick.endswith('3x3'):
+            return 'C0'
+        return 'C{}'.format(1+idx)
+
+    colors = [ get_color(i, n) for i, n in enumerate(df.nickname) ]
+    df.plot.scatter(ax=ax, x=perf_metric, y='accuracy', c=colors, logx=True)
 
     # Y axis
     ax.set_ylim(ylim)
     ax.set_ylabel('Accuracy')
+    ax.grid(True)
+    ax.tick_params(axis='y', grid_alpha=0.2, grid_color='black')
+
+    # X axis
+    ax.tick_params(axis='x', grid_alpha=0.0)
 
     if perf_metric == 'utilization':
         # mark feasible regions
@@ -135,20 +153,26 @@ def format_utilization(tick_val, tick_pos):
             return '{:d}%'.format(int(tick_val*100))
 
         ax.xaxis.set_major_formatter(matplotlib.ticker.FuncFormatter(format_utilization))
-        ax.set_xlabel('CPU utilization')
+        ax.set_xlabel('CPU usage')
 
     # Add markers
     def add_labels(row):
         xy = row[perf_metric], row.accuracy
         label = "{}".format(row.nickname) 
+        label = label.replace('Stride-DS-', 'S-DS-')
+        label = label.replace('Stride-', 'S-')
+
         ax.annotate(label, xy,
-                    xytext=(5,20),
+                    xytext=(2,5),
                     textcoords='offset points',
-                    size=10,
-                    rotation=25,
+                    rotation_mode='anchor',
+                    size=7,
+                    rotation=80,
                     color='darkslategrey')
     df.apply(add_labels, axis=1)
 
+    fig.tight_layout()
+
     return fig
 
 
@@ -231,12 +255,16 @@ def main():
 
     df['val_acc_mean'] = df.confusions_val.apply(get_accuracies).mean(axis=1)
     df['test_acc_mean'] = df.confusions_test.apply(get_accuracies).mean(axis=1)
+    df['test_acc_std'] = df.confusions_test.apply(get_accuracies).std(axis=1)
     df = df.sort_index()
 
     # TODO: add std-dev
     df['foreground_val_acc_mean'] = df.confusions_val_foreground.apply(get_accuracies).mean(axis=1)
     df['foreground_test_acc_mean'] = df.confusions_test_foreground.apply(get_accuracies).mean(axis=1)
     df['background_test_acc_mean'] = df.confusions_test_background.apply(get_accuracies).mean(axis=1)
+    df['foreground_val_acc_std'] = df.confusions_val_foreground.apply(get_accuracies).std(axis=1)
+    df['foreground_test_acc_std'] = df.confusions_test_foreground.apply(get_accuracies).std(axis=1)
+    df['background_test_acc_std'] = df.confusions_test_background.apply(get_accuracies).std(axis=1)
 
 
     #df['grouped_test_acc_mean'] = grouped_confusion(df.confusions_test, groups).apply(get_accuracies).mean(axis=1)
@@ -271,12 +299,12 @@ def save(fig, name):
 
 
     # Split the variations from all models
-    width_variations = df.nickname.str.startswith('Stride-DS-5x5-')
-    fig = plot_accuracy_comparison(df[width_variations != True])
+    width_variations = df.nickname.str.startswith('Stride-DS-')
+    fig = plot_accuracy_comparison(df[width_variations != True], ylim=(0.0, 1.0), figsize=(7,3))
     save(fig, 'models_accuracy.png')
 
     perf_metric = 'maccs_frame' if args.skip_device else 'utilization'
-    fig = plot_accuracy_vs_compute(df, perf_metric=perf_metric)
+    fig = plot_accuracy_vs_compute(df, perf_metric=perf_metric, figsize=(7,4), ylim=(0.5, 0.8))
     save(fig, 'models_efficiency.png')
 
 

models.csv

@@ -1,11 +1,11 @@
 model,conv_block,n_stages,conv_size,downsample_size,filters,modelcheck,nickname
 sbcnn,conv,3,5x5,3x2,24,skip,Baseline
 sbcnn,depthwise_separable,3,5x5,3x2,24,,Baseline-DS
-strided,conv,3,5x5,2x2,22,,Stride-5x5
-strided,depthwise_separable,3,5x5,2x2,24,,Stride-DS-5x5
-strided,effnet,3,5x5,2x2,22,,Stride-Effnet-5x5
+strided,conv,3,5x5,2x2,22,,Stride
+strided,depthwise_separable,3,5x5,2x2,24,,Stride-DS-24
+strided,effnet,3,5x5,2x2,22,,Stride-Effnet
 strided,depthwise_separable,4,3x3,2x2,24,,Stride-DS-3x3
-strided,bottleneck_ds,3,5x5,2x2,22,,Stride-BN-DS-5x5
-strided,depthwise_separable,3,5x5,2x2,20,,Stride-DS-5x5-20
-strided,depthwise_separable,3,5x5,2x2,16,,Stride-DS-5x5-16
-strided,depthwise_separable,3,5x5,2x2,12,,Stride-DS-5x5-12
+strided,bottleneck_ds,3,5x5,2x2,22,,Stride-BTLN-DS
+strided,depthwise_separable,3,5x5,2x2,20,,Stride-DS-20
+strided,depthwise_separable,3,5x5,2x2,16,,Stride-DS-16
+strided,depthwise_separable,3,5x5,2x2,12,,Stride-DS-12

report/plot.py

@@ -38,7 +38,7 @@ def model_table(data_path):
 
     return table.to_latex()
 
-def plot_models(data_path, figsize=(8,4), max_params=128e3, max_maccs=4.5e6):
+def plot_models(data_path, figsize=(12,4), max_params=128e3, max_maccs=4.5e6):
     df = logmel_models(data_path)
 
     fig, ax = plt.subplots(1, figsize=figsize)
@@ -49,7 +49,7 @@ def plot_models(data_path, figsize=(8,4), max_params=128e3, max_maccs=4.5e6):
 
     df.plot.scatter(x='params', y='macc_s', logx=True, logy=True, ax=ax)
     ax.set_xlabel('Model parameters')
-    ax.set_ylabel('Multiply-Adds / second')
+    ax.set_ylabel('MACC / second')
 
     # highlight feasible region
     feasible_x = max_params
@@ -73,6 +73,8 @@ def add_labels(row):
                     color='darkslategrey')
     df.apply(add_labels, axis=1)
 
+    fig.tight_layout()
+
     return fig
 
 urbansound8k_examples = {

report/plots/urbansound8k-existing-models-logmel.png

@@ -16,17 +16,15 @@
 def strformat(fmt, series):
     return [fmt.format(i) for i in series]
 
-#df = df.sort_values('maccs_frame', ascending=False)
 df = df.sort_values('nickname', ascending=True)
 
-width_multiple = df.nickname.str.startswith('Stride-DS-5x5-')
-df = df.loc[width_multiple == False]
+#width_multiple = df.nickname.str.startswith('Stride-DS-5x5-')
+#df = df.loc[width_multiple == False]
 
-#df = df.loc[[ df.nickname for ]] 
 
 conv_shorthand = {
     'depthwise_separable': 'DS',
-    'bottleneck_ds': 'BN-DS',
+    'bottleneck_ds': 'BTLN-DS',
     'effnet': 'Effnet',
     'conv': 'standard',
 }

report/pyincludes/models.py

@@ -1,13 +1,16 @@
 \begin{tabular}{lllrrlll}
 \toprule
-             Model &   Downsample & Convolution &  L &   F &     MACC &    RAM &   FLASH \\
+          Model &   Downsample & Convolution &  L &   F &     MACC &    RAM &   FLASH \\
 \midrule
-          Baseline &  maxpool 3x2 &    standard &  3 &  24 &  10185 K &  35 kB &  405 kB \\
-       Baseline-DS &  maxpool 3x2 &          DS &  3 &  24 &   1567 K &  55 kB &   96 kB \\
-        Stride-5x5 &   stride 2x2 &    standard &  3 &  22 &   2980 K &  55 kB &  372 kB \\
-  Stride-BN-DS-5x5 &   stride 2x2 &       BN-DS &  3 &  22 &    445 K &  47 kB &   80 kB \\
-     Stride-DS-3x3 &   stride 2x2 &          DS &  4 &  24 &    318 K &  54 kB &   95 kB \\
-     Stride-DS-5x5 &   stride 2x2 &          DS &  3 &  24 &    477 K &  54 kB &  180 kB \\
- Stride-Effnet-5x5 &   stride 2x2 &      Effnet &  3 &  22 &    468 K &  47 kB &  125 kB \\
+       Baseline &  maxpool 3x2 &    standard &  3 &  24 &  10185 K &  35 kB &  405 kB \\
+    Baseline-DS &  maxpool 3x2 &          DS &  3 &  24 &   1567 K &  55 kB &   96 kB \\
+         Stride &   stride 2x2 &    standard &  3 &  22 &   2980 K &  55 kB &  372 kB \\
+ Stride-BTLN-DS &   stride 2x2 &     BTLN-DS &  3 &  22 &    445 K &  47 kB &   80 kB \\
+   Stride-DS-12 &   stride 2x2 &          DS &  3 &  12 &    208 K &  27 kB &   88 kB \\
+   Stride-DS-16 &   stride 2x2 &          DS &  3 &  16 &    291 K &  36 kB &  118 kB \\
+   Stride-DS-20 &   stride 2x2 &          DS &  3 &  20 &    380 K &  45 kB &  149 kB \\
+   Stride-DS-24 &   stride 2x2 &          DS &  3 &  24 &    477 K &  54 kB &  180 kB \\
+  Stride-DS-3x3 &   stride 2x2 &          DS &  4 &  24 &    318 K &  54 kB &   95 kB \\
+  Stride-Effnet &   stride 2x2 &      Effnet &  3 &  22 &    468 K &  47 kB &  125 kB \\
 \bottomrule
 \end{tabular}

report/pyincludes/models.tex

@@ -7,18 +7,32 @@
 df = pandas.read_csv('results/results.csv')
 print(df)
 
-width_variations = df.nickname.str.startswith('Stride-DS-5x5-')
-df = df[width_variations != True]
+#width_variations = df.nickname.str.startswith('Stride-DS-5x5-')
+#df = df[width_variations != True]
+df = df.sort_values('nickname', ascending=True)
+
+def accuracies(df, col):
+    mean = df[col+'_mean'] * 100
+    std = df[col+'_std'] * 100
+
+    fmt = [ "{:.1f}% +-{:.1f}".format(*t) for t in zip(mean, std) ]
+    return fmt
+
+def cpu_use(df):
+    usage = (df.utilization * 1000 * 1/df.classifications_per_second).astype(int)
+    return ["{:d} ms".format(i).ljust(3) for i in usage]
 
 table = pandas.DataFrame({
     'Model': df.nickname,
-    'CPU (%)': (df.utilization * 100).astype(int),
-    'Accuracy': (df.test_acc_mean * 100).round(1),
-    'FG Accuracy': (df.foreground_test_acc_mean * 100).round(1),
-    'BG Accuracy': (df.background_test_acc_mean * 100).round(1),
+    'CPU use': cpu_use(df),
+    'Accuracy': accuracies(df, 'test_acc'),
+    'FG Accuracy': accuracies(df, 'foreground_test_acc'),
+    'BG Accuracy': accuracies(df, 'background_test_acc'),
 }, index=df.index)
 
-out = table.to_latex(header=True, index=False)
+
+out = table.to_latex(header=True, index=False, column_format='lrrrr')
+out = out.replace('+-', '$\pm$') # XXX: Latex gets mangled by to_table it seems
 print(out)
 
 outpath = sys.argv[1] 

report/pyincludes/results.py

@@ -1,13 +1,16 @@
 \begin{tabular}{lrrrr}
 \toprule
-             Model &  CPU (\%) &  Accuracy &  FG Accuracy &  BG Accuracy \\
+          Model & CPU use &     Accuracy &  FG Accuracy &   BG Accuracy \\
 \midrule
-          Baseline &        0 &      73.1 &         84.3 &         49.6 \\
-        Stride-5x5 &        0 &      71.9 &         82.1 &         50.2 \\
- Stride-Effnet-5x5 &        0 &      67.1 &         75.5 &         49.5 \\
-     Stride-DS-5x5 &        0 &      72.5 &         81.3 &         54.0 \\
-     Stride-DS-3x3 &        0 &      70.1 &         79.4 &         50.6 \\
-  Stride-BN-DS-5x5 &        0 &      68.6 &         77.7 &         49.3 \\
-       Baseline-DS &        0 &      72.7 &         84.0 &         48.8 \\
+       Baseline &  971 ms &  72.3\% $\pm$4.6 &  78.3\% $\pm$7.1 &   60.5\% $\pm$7.7 \\
+    Baseline-DS &  244 ms &  70.2\% $\pm$4.7 &  76.1\% $\pm$7.5 &   58.6\% $\pm$8.2 \\
+         Stride &  325 ms &  68.3\% $\pm$5.2 &  74.1\% $\pm$6.6 &   56.6\% $\pm$8.0 \\
+ Stride-BTLN-DS &   71 ms &  64.8\% $\pm$7.1 &  69.5\% $\pm$8.2 &   55.3\% $\pm$8.9 \\
+   Stride-DS-12 &   38 ms &  66.0\% $\pm$6.0 &  72.6\% $\pm$6.5 &   53.3\% $\pm$9.1 \\
+   Stride-DS-16 &   51 ms &  67.5\% $\pm$5.6 &  73.3\% $\pm$7.7 &   56.2\% $\pm$8.3 \\
+   Stride-DS-20 &   66 ms &  68.4\% $\pm$5.2 &  75.0\% $\pm$7.4 &  55.2\% $\pm$10.0 \\
+   Stride-DS-24 &   81 ms &  70.9\% $\pm$4.3 &  75.8\% $\pm$6.3 &   61.8\% $\pm$6.8 \\
+  Stride-DS-3x3 &   59 ms &  67.2\% $\pm$6.5 &  73.0\% $\pm$7.4 &   55.8\% $\pm$9.1 \\
+  Stride-Effnet &   73 ms &  60.7\% $\pm$6.6 &  66.9\% $\pm$7.9 &   48.7\% $\pm$8.3 \\
 \bottomrule
 \end{tabular}