feat: add show_hard_limit option and update time limit handling in DataScience settings (#1144)

Author: peteryang1

rdagent/app/data_science/conf.py

@@ -126,6 +126,7 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     coder_longer_timeout_multiplier_upper: int = 3
     runner_longer_timeout_multiplier_upper: int = 2
     timeout_increase_stage: float = 0.3
+    show_hard_limit: bool = True
 
     #### hypothesis critique and rewrite
     enable_hypo_critique_rewrite: bool = True

rdagent/scenarios/data_science/scen/__init__.py

@@ -180,7 +180,6 @@ def get_competition_full_desc(self) -> str:
             time_limit=None,
             recommend_time_limit=None,
             eda_output=None,
-            sample_data_by_LLM=None,
             debug_time_limit=None,
             recommend_debug_time_limit=None,
             runtime_environment=self.get_runtime_environment(),
@@ -198,12 +197,17 @@ def get_scenario_all_desc(self, eda_output=None) -> str:
             metric_direction=self.metric_direction,
             raw_description=self.raw_description,
             use_raw_description=DS_RD_SETTING.use_raw_description,
-            time_limit=f"{self.real_full_timeout() / 60 / 60 : .2f} hours",
-            recommend_time_limit=f"{self.recommend_full_timeout() / 60 / 60 : .2f} hours",
+            time_limit=f"{self.real_full_timeout() / 60 / 60 : .2f} hours" if DS_RD_SETTING.show_hard_limit else None,
+            recommend_time_limit=(
+                f"{self.recommend_full_timeout() / 60 / 60 : .2f} hours" if DS_RD_SETTING.sample_data_by_LLM else None
+            ),
             eda_output=eda_output,
-            sample_data_by_LLM=DS_RD_SETTING.sample_data_by_LLM,
-            debug_time_limit=f"{self.real_debug_timeout() / 60 : .2f} minutes",
-            recommend_debug_time_limit=f"{self.recommend_debug_timeout() / 60 : .2f} minutes",
+            debug_time_limit=(
+                f"{self.real_debug_timeout() / 60 : .2f} minutes" if DS_RD_SETTING.show_hard_limit else None
+            ),
+            recommend_debug_time_limit=(
+                f"{self.recommend_debug_timeout() / 60 : .2f} minutes" if DS_RD_SETTING.sample_data_by_LLM else None
+            ),
             runtime_environment=self.get_runtime_environment(),
         )
 

rdagent/scenarios/data_science/scen/prompts.yaml

@@ -34,21 +34,20 @@ scenario_description: |-
   {{ evaluation }}
   {% endif %}
 
-  {% if time_limit %}
+  {% if time_limit is not none %}
   ====== Time Limit On Full Code Execution ======
   Your full code's execution is limited to **{{ time_limit }}**. After this time limit, your code will be terminated and all time and resources are wasted. Always make sure your code will not run longer than this time limit.
   During this time limit, you have all the resources available to you. Please fully leverage all the computational resources(CPUs and GPUs) to achieve the best performance like choose a powerful model, use a large batch size, enable data sampler with big parallel.
-  {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %}
+  {% endif %}{% if debug_time_limit is not none%}
   ====== Time Limit On Debug Mode Code Execution ======
   Your are also required to include a debug mode in your code, the debug code's execution is limited to **{{ debug_time_limit }}**. You should make sure 10 percent of the data training one epoch can be finished within this time limit. If not, your should propose a new debug strategy in your task.
-  {% endif %}
+  {% endif %}{% if recommend_time_limit is not none %}
   ====== Recommend Time Spent On Full Code Execution ======
   You should always prioritize performance over time spent since some tasks requires very little time to run the code to achieve the best performance while some tasks might need a lot of time to train one or more large models.
-  We recommend you to spend less than **{{ recommend_time_limit }}** on the full code execution to boost efficiency. This is a recommended time limit, you can spend more time on the code execution if you think it is necessary. But always remember to not exceed the time limit.
-  {% if sample_data_by_LLM is not none and sample_data_by_LLM is true %}
+  We recommend you to spend less than **{{ recommend_time_limit }}** on the full code execution to boost efficiency. This is a recommended time limit, you can spend more time on the code execution if you think it is very necessary. But your code could be terminated sometime after this recommend time limit.
+  {% endif %}{% if recommend_debug_time_limit is not none %}
   ====== Recommend Time Spent On Debug Mode Code Execution ======
-  We recommend you to spend less than **{{ recommend_debug_time_limit }}** on the debug mode code execution to boost efficiency. This is a recommended time limit, you can spend more time on the debug mode code execution if you think it is necessary. But always remember to not exceed the time limit.
-  {% endif %}
+  We recommend you to spend less than **{{ recommend_debug_time_limit }}** on the debug mode code execution to boost efficiency. This is a recommended time limit, you can spend more time on the debug mode code execution if you think it is very necessary. But your code could be terminated sometime after this recommend time limit.
   {% endif %}
 
   {% if runtime_environment is not none %}

rdagent/utils/__init__.py

@@ -127,7 +127,7 @@ def filter_redundant_text(stdout: str) -> str:
     for line in filtered_stdout_lines:
         lines_to_count[line] = lines_to_count.get(line, 0) + 1
     filtered_stdout = "\n".join(
-        [line for line in filtered_stdout_lines if lines_to_count[line] <= len(filtered_stdout_lines) // 10]
+        [line for line in filtered_stdout_lines if lines_to_count[line] <= max(len(filtered_stdout_lines) // 10, 10)]
     )
 
     # Iteratively ask the LLM for additional filtering patterns (up to 3 rounds)