Improve Benchmarking Performance (#222)

Author: FANGAreNotGnu

.gitignore

@@ -2,7 +2,7 @@
 maab/datasets/
 runs/
 maab/runs/
-/maab/
+temp/
 
 # OS specific
 *.DS_Store

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
   "langchain-anthropic>=0.3.15",
   "langchain_aws>=0.2.2",
   "pydantic>=2.9.2",
-  "hydra-core>=1.3",
+  "hydra-core",
   "matplotlib>=3.9.2",
   "typer>=0.12.5",
   "rich>=13.8.1",
@@ -35,6 +35,8 @@ dependencies = [
   "joblib>=1.4.2",
   "python-calamine",
   "tenacity>=8.2.2",
+  "torchaudio",
+  "torchvision",
   "transformers<=4.49.0",
   "pandas>=2.2",
   "streamlit>=1.37",

src/autogluon/assistant/agents/reranker_agent.py

@@ -110,7 +110,8 @@ def _generate_tutorial_prompt(self, selected_tutorials: List[TutorialInfo]) -> s
         # Format selected tutorials
         formatted_tutorials = []
         for tutorial in selected_tutorials:
-            formatted = self._format_tutorial_content(tutorial, max_tutorial_length)
+            per_tutorial_max_length = max_tutorial_length // len(selected_tutorials)
+            formatted = self._format_tutorial_content(tutorial, per_tutorial_max_length)
             if formatted:
                 formatted_tutorials.append(formatted)
 

src/autogluon/assistant/configs/default.yaml

@@ -7,11 +7,11 @@ max_file_group_size_to_show: 5
 num_example_files_to_show: 1
 
 max_chars_per_file: 1024
-num_tutorial_retrievals: 20
+num_tutorial_retrievals: 30
 max_num_tutorials: 5
 max_user_input_length: 2048
 max_error_message_length: 2048
-max_tutorial_length: 8192
+max_tutorial_length: 32768
 create_venv: false
 condense_tutorials: True
 use_tutorial_summary: True
@@ -27,19 +27,18 @@ llm: &default_llm
   #model: gpt-4o-2024-08-06
   #provider: anthropic
   # model: claude-3-7-sonnet-20250219
-  max_tokens: 16384
+  max_tokens: 32768
   proxy_url: null
-  temperature: 1.
+  temperature: 0.1
+  top_p: 0.9
   verbose: True
   multi_turn: False
   template: null
   add_coding_format_instruction: false
 
 coder:
   <<: *default_llm  # Merge llm_config
-  temperature: 0.
   multi_turn: True
-  top_p: 1
 
 executer:
   <<: *default_llm  # Merge llm_config
@@ -58,9 +57,13 @@ retriever:
 
 reranker:
   <<: *default_llm  # Merge llm_config
+  temperature: 0.
+  top_p: 1.
 
 description_file_retriever:
   <<: *default_llm  # Merge llm_config
+  temperature: 0.
+  top_p: 1.
 
 task_descriptor:
   <<: *default_llm  # Merge llm_config
@@ -69,3 +72,5 @@ task_descriptor:
 
 tool_selector:
   <<: *default_llm  # Merge llm_config
+  temperature: 0.
+  top_p: 1.

src/autogluon/assistant/managers/manager.py

@@ -154,6 +154,10 @@ def generate_initial_prompts(self):
 
         self.selected_tool = self.ts_agent()
 
+        # TODO: remove the hard code for "create_venv" (add in tool registry if need installation)
+        if self.selected_tool.lower() in ["machine learning", "huggingface", "fairseq"]:
+            self.config.create_venv = True
+
         # Get tool-specific template and requirements if they exist
         tool_info = registry.get_tool(self.selected_tool)
         if not tool_info:

src/autogluon/assistant/prompts/bash_coder_prompt.py

@@ -75,8 +75,6 @@ def parse(self, response: Dict) -> Tuple[str, Optional[str]]:
 
     def get_env_prompt(self):
         create_venv = self.manager.config.create_venv
-        # TODO: remove the hard code for "install_packages" (add in tool registry if need installation)
-        install_packages = "machine learning" in self.manager.selected_tool
         output_folder = self.manager.output_folder
         selected_tool = self.manager.selected_tool
 
@@ -87,10 +85,6 @@ def get_env_prompt(self):
     - Python version: 3.11
     - Activate the environment
     - Install required packages"""
-        elif install_packages:
-            env_prompt = (
-                "The environment may not be fully configured. Install any packages required in the python code."
-            )
         else:
             env_prompt = f"The environment is already configured. Do not install or update any package unless there is an error due to the missing package. \nDo NOT upgrade {selected_tool} which is already installed."
 

src/autogluon/assistant/prompts/python_coder_prompt.py

@@ -88,6 +88,20 @@ def build(self) -> str:
             )
             prompt = f"{prompt}\n\n{format_instruction}"
 
+        # TODO: Remove hardcoding. And add this safeguard for other prompts.
+        if len(prompt) > 100000:
+            logger.warning(f"Coder's prompt too long: {len(prompt)}. Truncated.")
+            self.manager.save_and_log_states(
+                content=prompt,
+                save_name="python_coder_prompt_before_truncation.txt",
+                per_iteration=True,
+                add_uuid=False,
+            )
+            prompt = self._truncate_output_end(
+                output=prompt,
+                max_length=100000,
+            )
+
         self.manager.save_and_log_states(
             content=prompt, save_name="python_coder_prompt.txt", per_iteration=True, add_uuid=False
         )

src/autogluon/assistant/prompts/task_descriptor_prompt.py

@@ -14,6 +14,8 @@ def default_template(self) -> str:
         return """
 Based ONLY on the information explicitly stated in the provided data structure and description files, provide a condensed and precise description of the data science task. Include only details that are directly mentioned in the source materials. Do not add assumptions or infer unstated information.
 
+Be very clear about the problem type (e.g. audio classification/image regression/seq-to-seq generation/etc.), input format, and prediction output format.
+
 ### Data Structure:
 (IMPORTANT: The metadata of example files in Data Structure may not be representative - do not make assumptions about data statistics based on examples.)
 {data_prompt}

src/autogluon/assistant/tools_registry/FlagEmbedding/tool.json

@@ -1,10 +1,8 @@
 {
   "name": "FlagEmbedding",
   "version": "1.3.4",
-  "description": "Retrieval and Retrieval-augmented LLMs",
+  "description": "FlagEmbedding specializes in retrieval and reranking tasks.",
   "features": [
-    "retrieval",
-    "reranking"
   ],
   "requirements": [],
   "prompt_template": [

src/autogluon/assistant/tools_registry/_common/catalog.json

@@ -3,27 +3,37 @@
     "autogluon.timeseries": {
       "path": "autogluon.timeseries",
       "version": "1.2.0",
-      "description": "AutoGluon Timeseries is an open-source AutoML framework that automates the training and tuning of forecasting models for time series data, handling tasks from preprocessing to model ensembling with built-in support for both univariate and multivariate forecasting."
+      "description": "AutoGluon TimeSeries automates forecasting model training for time series data, supporting both univariate and multivariate predictions."
     },
     "autogluon.multimodal": {
       "path": "autogluon.multimodal",
       "version": "1.2.0",
-      "description": "AutoGluon Multimodal is an open-source AutoML framework that simplifies the training of models across multiple data types including text, images, and tabular data, automating tasks from preprocessing to model ensembling with minimal code required."
+      "description": "AutoGluon Multimodal excels at multimodal classification, regression, document classification, and semantic segmentation tasks. The framework is not optimized for pure tabular data processing and doesn't support generative tasks (like image-to-image or sequence-to-sequence) or audio-related tasks. It also doesn't handle object detection."
     },
     "autogluon.tabular": {
       "path": "autogluon.tabular",
       "version": "1.2.0",
-      "description": "AutoGluon Tabular is an open-source AutoML framework that automates the training and tuning of machine learning models for tabular data, handling tasks from preprocessing to model ensembling with minimal code required."
+      "description": "AutoGluon Tabular automates ML model training for tabular data, excelling at numerical and categorical data processing. It doesn't support NLP or image-based tasks."
     },
     "FlagEmbedding": {
       "path": "FlagEmbedding",
       "version": "1.3.4",
-      "description": "Retrieval and Retrieval-augmented LLMs"
+      "description": "FlagEmbedding specializes in retrieval and reranking tasks."
     },
-    "machine learning or deep learning": {
+    "machine learning": {
       "path": "machine learning",
       "version": "0.1.0",
       "description": "You should select this as a general reference of machine learning or deep learning algorithms in case other tools are not helpful."
+    },
+    "wav2vec2": {
+      "path": "wav2vec2",
+      "version": "0.0.0",
+      "description": "This model was pre-trained on 4.5M hours of unlabeled audio data covering more than 143 languages. It requires finetuning to be used for downstream tasks such as Automatic Speech Recognition (ASR), or Audio Classification."
+    },
+    "qwen3": {
+      "path": "qwen3",
+      "version": "0.0.0",
+      "description": "Qwen3-0.6B is a compact 0.6 billion parameter text generation model that uniquely switches between thinking and non-thinking modes for both efficient dialogue and complex reasoning tasks."
     }
   }
 }