chore: overwrite sota exp stat in summary page (#1146)

* overwrite sota exp stat in summary page

* remove unused import

Author: XianBW

rdagent/log/mle_summary.py

@@ -24,23 +24,21 @@
 def save_grade_info(log_trace_path: Path):
     test_eval = get_test_eval()
 
-    is_mle = isinstance(test_eval, MLETestEval)
     trace_storage = FileStorage(log_trace_path)
-    for msg in trace_storage.iter_msg():
-        if "competition" in msg.tag:
-            competition = msg.content
+    for msg in trace_storage.iter_msg(tag="competition"):
+        competition = msg.content
 
-        if "running" in msg.tag:
-            if isinstance(msg.content, DSExperiment):
-                # TODO:  mle_score.txt is not a general name now.
-                # Please use a more general name like test_score.txt
-                try:
-                    mle_score_str = test_eval.eval(competition, msg.content.experiment_workspace)
-                    trace_storage.log(
-                        mle_score_str, tag=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
-                    )
-                except Exception as e:
-                    print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
+    for msg in trace_storage.iter_msg(tag="running"):
+        if isinstance(msg.content, DSExperiment):
+            # TODO:  mle_score.txt is not a general name now.
+            # Please use a more general name like test_score.txt
+            try:
+                mle_score_str = test_eval.eval(competition, msg.content.experiment_workspace)
+                trace_storage.log(
+                    mle_score_str, tag=f"{msg.tag}.mle_score.pid", save_type="pkl", timestamp=msg.timestamp
+                )
+            except Exception as e:
+                print(f"Error in {log_trace_path}: {e}", traceback.format_exc())
 
 
 def save_all_grade_info(log_folder: str | Path) -> None:

rdagent/log/ui/utils.py

@@ -313,16 +313,22 @@ def get_summary_df(log_folders: list[str], hours: int | None = None) -> tuple[di
             v["coding_time"] = str(coding_time).split(".")[0]
             v["running_time"] = str(running_time).split(".")[0]
 
-            sota_exp, v["sota_loop_id_new"], sota_report, v["sota_exp_stat_new"] = get_sota_exp_stat(
+            # overwrite sota_exp_stat in summary.pkl because it may not be correct in multi-trace
+            _, _, sota_report, v["sota_exp_stat"] = get_sota_exp_stat(Path(lf) / k, to_submit=False)
+            v["sota_exp_score"] = sota_report["score"] if sota_report else None
+
+            sota_exp_submit, v["sota_loop_id_new"], sota_submit_report, v["sota_exp_stat_new"] = get_sota_exp_stat(
                 Path(lf) / k, to_submit=True
             )
-            if sota_exp is not None:
+            if sota_exp_submit is not None:
                 try:
-                    sota_result = sota_exp.result
+                    sota_submit_result = sota_exp_submit.result
                 except AttributeError:  # Compatible with old versions
-                    sota_result = sota_exp.__dict__["result"]
-                v["sota_exp_score_valid_new"] = sota_result.loc["ensemble"].iloc[0] if sota_result is not None else None
-            v["sota_exp_score_new"] = sota_report["score"] if sota_report else None
+                    sota_submit_result = sota_exp_submit.__dict__["result"]
+                v["sota_exp_score_valid_new"] = (
+                    sota_submit_result.loc["ensemble"].iloc[0] if sota_submit_result is not None else None
+                )
+            v["sota_exp_score_new"] = sota_submit_report["score"] if sota_submit_report else None
             # change experiment name
             if "amlt" in lf:
                 summary[f"{lf[lf.rfind('amlt')+5:].split('/')[0]} - {k}"] = v