add option to choose "worst" line by maximum wer area under the curve

Author: SanderGi

notebooks/Eval/ASR_eval.ipynb

@@ -760,7 +760,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 50,
    "id": "a11013dc",
    "metadata": {},
    "outputs": [
@@ -947,6 +947,7 @@
     "REQUIRE_NATIVE = True\n",
     "# WORST_CRITERIA = 'max_regression'\n",
     "WORST_CRITERIA = 'avg_regression'\n",
+    "# WORST_CRITERIA = 'area'\n",
     "INCLUDE_UNCONDITIONAL = False\n",
     "PLOT_TOP_TEN_DIALECTS = True\n",
     "# STATISTIC = 'median'\n",
@@ -1029,7 +1030,16 @@
     "    # Convert to DataFrame and print\n",
     "    reg_df = pd.DataFrame(res)\n",
     "\n",
-    "    top_lang = reg_df.loc[reg_df[WORST_CRITERIA].idxmax(), 'native_language']\n",
+    "    if WORST_CRITERIA == 'area':\n",
+    "        top_lang = None\n",
+    "        max_area = -1\n",
+    "        for l in reg_df['native_language'].unique():\n",
+    "            area = df_d[df_d['native_language'] == l].groupby(['release_date', 'model']).agg({'wer': STATISTIC}).sum()['wer']\n",
+    "            if area > max_area:\n",
+    "                top_lang = l\n",
+    "                max_area = area\n",
+    "    else:\n",
+    "        top_lang = reg_df.loc[reg_df[WORST_CRITERIA].idxmax(), 'native_language']\n",
     "    if REQUIRE_NATIVE and not (native_lang and native_lang != top_lang):\n",
     "        continue\n",
     "\n",